diff --git a/.gitattributes b/.gitattributes index d2178eded02f4d8450f705b37ac8d57da1663441..e049502a8747cf91fc77ce26f22ecf4f6924a342 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,3 +34,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints/iter_0006855/.metadata filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_013957-a2kv2ydx/run-a2kv2ydx.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_010425-k7i86mla/run-k7i86mla.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_015603-9a1n96nr/run-9a1n96nr.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_024051-df4j15aa/run-df4j15aa.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_021843-lzgz3u05/run-lzgz3u05.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261010_231200-gdb9n0eb/run-gdb9n0eb.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/run-acr8mx9p.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0001.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0001.json new file mode 100644 index 0000000000000000000000000000000000000000..2f674ba3055f9964efba3d08f801bb36272cc844 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0001.json @@ -0,0 +1,84 @@ +{ + "rank": 1, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0002.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0002.json new file mode 100644 index 0000000000000000000000000000000000000000..71ca2411e7a86bebce31a69f7f9d1ae1cedeadd2 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0002.json @@ -0,0 +1,84 @@ +{ + "rank": 2, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0004.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0004.json new file mode 100644 index 0000000000000000000000000000000000000000..a1ab57676cbfebc565044d99e9f49e5e13b33ec6 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0004.json @@ -0,0 +1,84 @@ +{ + "rank": 4, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0006.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0006.json new file mode 100644 index 0000000000000000000000000000000000000000..55f5a514f244049ca32b20911eb1162518e02032 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0006.json @@ -0,0 +1,84 @@ +{ + "rank": 6, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0012.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0012.json new file mode 100644 index 0000000000000000000000000000000000000000..b01a7a2f939329681668180a8ec819502d22259e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0012.json @@ -0,0 +1,84 @@ +{ + "rank": 12, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0013.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0013.json new file mode 100644 index 0000000000000000000000000000000000000000..0e34445041a60ab35f3a095b27a3108bd4cc112a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0013.json @@ -0,0 +1,84 @@ +{ + "rank": 13, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0015.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0015.json new file mode 100644 index 0000000000000000000000000000000000000000..c54b66bbcf5acdd8472f7d5029bd2f4b80795664 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0015.json @@ -0,0 +1,84 @@ +{ + "rank": 15, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0026.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0026.json new file mode 100644 index 0000000000000000000000000000000000000000..a0f2afb6fd7ddc2040fd267572253b02ef3dea6c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0026.json @@ -0,0 +1,84 @@ +{ + "rank": 26, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0027.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0027.json new file mode 100644 index 0000000000000000000000000000000000000000..083228c0d89ee1b11e764a1d1a3b0b15da0bd3b5 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0027.json @@ -0,0 +1,84 @@ +{ + "rank": 27, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0028.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0028.json new file mode 100644 index 0000000000000000000000000000000000000000..15bec66beab6c1eb26660fd9d231982858d58e01 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0028.json @@ -0,0 +1,84 @@ +{ + "rank": 28, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0030.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0030.json new file mode 100644 index 0000000000000000000000000000000000000000..03e670709ef91418bba8e43f4d6dcc8b4e10c890 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0030.json @@ -0,0 +1,84 @@ +{ + "rank": 30, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0033.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0033.json new file mode 100644 index 0000000000000000000000000000000000000000..dd06ecf3c7691d018cefac65148a0e73a368121e --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0033.json @@ -0,0 +1,84 @@ +{ + "rank": 33, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0035.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0035.json new file mode 100644 index 0000000000000000000000000000000000000000..4c2bea10ab61f76b9fc8133bbf7dac1f476f2969 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0035.json @@ -0,0 +1,84 @@ +{ + "rank": 35, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0036.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0036.json new file mode 100644 index 0000000000000000000000000000000000000000..beb4630fc71950a1149edd8901a55b99cbeb5eb1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0036.json @@ -0,0 +1,84 @@ +{ + "rank": 36, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0037.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0037.json new file mode 100644 index 0000000000000000000000000000000000000000..cea4b5204c838ffa2dce9f3601e8319e84834e6d --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0037.json @@ -0,0 +1,84 @@ +{ + "rank": 37, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0042.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0042.json new file mode 100644 index 0000000000000000000000000000000000000000..1c0768f68addb974a73b27956b1f5edfd6c64481 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0042.json @@ -0,0 +1,84 @@ +{ + "rank": 42, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0050.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0050.json new file mode 100644 index 0000000000000000000000000000000000000000..c20b4b44fbc71ec67f54b24aaae27cd35f794d1f --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0050.json @@ -0,0 +1,84 @@ +{ + "rank": 50, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0058.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0058.json new file mode 100644 index 0000000000000000000000000000000000000000..599ded4d8ff545314d058e621ad5e07f4c67459b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0058.json @@ -0,0 +1,84 @@ +{ + "rank": 58, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0059.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0059.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea93f2fcecc564d126d471384ffa4a4b9b1b201 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0059.json @@ -0,0 +1,84 @@ +{ + "rank": 59, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0005.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0005.json new file mode 100644 index 0000000000000000000000000000000000000000..979022858d2ed2e1bc905b457a482f9db21ffac3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0005.json @@ -0,0 +1,13227 @@ +{ + "rank": 5, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0006.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0006.json new file mode 100644 index 0000000000000000000000000000000000000000..fd9d29f5157a01b992939bf8c58372b0eab7be48 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0006.json @@ -0,0 +1,13227 @@ +{ + "rank": 6, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0007.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0007.json new file mode 100644 index 0000000000000000000000000000000000000000..611dd8e2c26c9cf6d4dbd789e5dce59a209f3286 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0007.json @@ -0,0 +1,13227 @@ +{ + "rank": 7, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0008.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0008.json new file mode 100644 index 0000000000000000000000000000000000000000..37760940e5c0894fef7c092cdc1d5c0effc5cd13 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0008.json @@ -0,0 +1,13227 @@ +{ + "rank": 8, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0009.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0009.json new file mode 100644 index 0000000000000000000000000000000000000000..546ca62d2b1c59e18154b5c6ce64f28b9eb47e80 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0009.json @@ -0,0 +1,13227 @@ +{ + "rank": 9, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0011.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0011.json new file mode 100644 index 0000000000000000000000000000000000000000..7eed4324bc97ce2c80e292fcc03b5f97d6d84573 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0011.json @@ -0,0 +1,13227 @@ +{ + "rank": 11, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0016.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0016.json new file mode 100644 index 0000000000000000000000000000000000000000..efa9c84b0e02e60537398954642cf0b0c2a3bca9 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0016.json @@ -0,0 +1,13227 @@ +{ + "rank": 16, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0021.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0021.json new file mode 100644 index 0000000000000000000000000000000000000000..630766cfc3a50b901eb50b65fe313d29bad27537 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0021.json @@ -0,0 +1,13227 @@ +{ + "rank": 21, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0023.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0023.json new file mode 100644 index 0000000000000000000000000000000000000000..1fb5b38025e65473c439e7cb586bf621b753afbd --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0023.json @@ -0,0 +1,13227 @@ +{ + "rank": 23, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0024.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0024.json new file mode 100644 index 0000000000000000000000000000000000000000..2caf7e9638abfd6cfb484cdd276c02d941033718 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0024.json @@ -0,0 +1,13227 @@ +{ + "rank": 24, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0028.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0028.json new file mode 100644 index 0000000000000000000000000000000000000000..63d80993b24178f334a1e98eed3edf2356969a18 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0028.json @@ -0,0 +1,13227 @@ +{ + "rank": 28, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0030.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0030.json new file mode 100644 index 0000000000000000000000000000000000000000..32e4243e61ef7f63a27f4856ca65d8f0d3cfc3fd --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0030.json @@ -0,0 +1,13227 @@ +{ + "rank": 30, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0039.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0039.json new file mode 100644 index 0000000000000000000000000000000000000000..aa12eab606bb62d7533451411ab8415cb5ec8420 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0039.json @@ -0,0 +1,13227 @@ +{ + "rank": 39, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0040.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0040.json new file mode 100644 index 0000000000000000000000000000000000000000..d831305eb75faf5989cde4e63f5620ed8d91c7e6 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0040.json @@ -0,0 +1,13227 @@ +{ + "rank": 40, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0044.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0044.json new file mode 100644 index 0000000000000000000000000000000000000000..a62c1de5cc9a54ab7d95c594ece32914e59e754b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0044.json @@ -0,0 +1,13227 @@ +{ + "rank": 44, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0053.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0053.json new file mode 100644 index 0000000000000000000000000000000000000000..02bb171c51da6ce1b897631e499347030a28a5d9 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0053.json @@ -0,0 +1,13227 @@ +{ + "rank": 53, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0054.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0054.json new file mode 100644 index 0000000000000000000000000000000000000000..9db2c584f0cb3b0ef07447d7732373b37d1894ca --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0054.json @@ -0,0 +1,13227 @@ +{ + "rank": 54, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0055.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0055.json new file mode 100644 index 0000000000000000000000000000000000000000..f8db73cfe512c66e55cbd58672a4fbe11cea5770 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0055.json @@ -0,0 +1,13227 @@ +{ + "rank": 55, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0057.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0057.json new file mode 100644 index 0000000000000000000000000000000000000000..8f7dd4eddfa95ab0b3a6c07c39da809fd42f876c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0057.json @@ -0,0 +1,13227 @@ +{ + "rank": 57, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0058.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0058.json new file mode 100644 index 0000000000000000000000000000000000000000..868c454bc12dd7450792250733134e3ce516d388 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0058.json @@ -0,0 +1,13227 @@ +{ + "rank": 58, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0059.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0059.json new file mode 100644 index 0000000000000000000000000000000000000000..b8e5906bf28d476d3ae7c11675254c1192852b43 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0059.json @@ -0,0 +1,13227 @@ +{ + "rank": 59, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0060.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0060.json new file mode 100644 index 0000000000000000000000000000000000000000..2f7857e98da57bec90df0bfae786449e86c93481 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0060.json @@ -0,0 +1,13227 @@ +{ + "rank": 60, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/optimizer0_rank0000.jsonl b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/optimizer0_rank0000.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..50fa5be99e70387b782ef72b9970cf2b4c909a56 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/optimizer0_rank0000.jsonl @@ -0,0 +1,2404 @@ +{"call": 1, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.891949569058555e-08, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.08901979494839907} +{"call": 2, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027764974860474467} +{"call": 3, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028064655838534236} +{"call": 4, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027496488066390157} +{"call": 5, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027485221158713102} +{"call": 6, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03143748617731035} +{"call": 7, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02760343300178647} +{"call": 8, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027638813015073538} +{"call": 9, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027679253136739135} +{"call": 10, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.533381818207865e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047381892101839185} +{"call": 11, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.034424839075654745} +{"call": 12, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027948179049417377} +{"call": 13, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02803990594111383} +{"call": 14, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028536309022456408} +{"call": 15, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027944857953116298} +{"call": 16, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030990012921392918} +{"call": 17, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02806740184314549} +{"call": 18, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029970004921779037} +{"call": 19, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029990077018737793} +{"call": 20, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.536398412084116e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05040170508436859} +{"call": 21, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027689953101798892} +{"call": 22, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028037202078849077} +{"call": 23, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03015814209356904} +{"call": 24, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027610238874331117} +{"call": 25, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028260807041078806} +{"call": 26, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027912233024835587} +{"call": 27, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02897909702733159} +{"call": 28, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030051248846575618} +{"call": 29, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028657476883381605} +{"call": 30, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.727213854939451e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04994151508435607} +{"call": 31, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02804988482967019} +{"call": 32, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04653931292705238} +{"call": 33, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030693134060129523} +{"call": 34, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029198682866990566} +{"call": 35, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027906248811632395} +{"call": 36, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027647888055071235} +{"call": 37, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027548243990167975} +{"call": 38, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031282583018764853} +{"call": 39, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775519690476358} +{"call": 40, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.572464253116826e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049665097845718265} +{"call": 41, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027936830883845687} +{"call": 42, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02791413594968617} +{"call": 43, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772795083001256} +{"call": 44, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030096312053501606} +{"call": 45, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02853217301890254} +{"call": 46, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027860145084559917} +{"call": 47, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027945354813709855} +{"call": 48, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02960479399189353} +{"call": 49, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0280375259462744} +{"call": 50, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.177613630081944e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05021744081750512} +{"call": 51, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02768627880141139} +{"call": 52, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033180342987179756} +{"call": 53, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028341061901301146} +{"call": 54, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028012472903355956} +{"call": 55, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027729223016649485} +{"call": 56, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030575677985325456} +{"call": 57, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029731726041063666} +{"call": 58, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028122605988755822} +{"call": 59, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027735227020457387} +{"call": 60, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.61929262275645e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049854499055072665} +{"call": 41, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04910415201447904} +{"call": 42, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027306878939270973} +{"call": 43, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029553564032539725} +{"call": 44, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03238786803558469} +{"call": 45, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027617996791377664} +{"call": 46, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027663041139021516} +{"call": 47, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029423952801153064} +{"call": 48, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027397326892241836} +{"call": 49, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027672455878928304} +{"call": 50, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.0218827226573887e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05081739090383053} +{"call": 51, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030834618024528027} +{"call": 52, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02882698201574385} +{"call": 53, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03270749398507178} +{"call": 54, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028371990891173482} +{"call": 55, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027328999945893884} +{"call": 56, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029284006915986538} +{"call": 57, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027348595904186368} +{"call": 58, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027613725047558546} +{"call": 59, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029413263080641627} +{"call": 60, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.7952973107493006e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04905725992284715} +{"call": 61, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02732242993079126} +{"call": 62, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029854049906134605} +{"call": 63, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028602255042642355} +{"call": 64, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028354797046631575} +{"call": 65, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031185227911919355} +{"call": 66, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028660947922617197} +{"call": 67, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029805079102516174} +{"call": 68, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028949858155101538} +{"call": 69, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027125868014991283} +{"call": 70, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.8372612925378483e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048727137967944145} +{"call": 71, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027419345919042826} +{"call": 72, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027365149930119514} +{"call": 73, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03192488895729184} +{"call": 74, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027618815191090107} +{"call": 75, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027310593985021114} +{"call": 76, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02853391505777836} +{"call": 77, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03048360999673605} +{"call": 78, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028381444048136473} +{"call": 79, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028516486985608935} +{"call": 80, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.826170256581102e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048816462978720665} +{"call": 81, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02780531393364072} +{"call": 82, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02753606205806136} +{"call": 83, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027869366109371185} +{"call": 84, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028208223870024085} +{"call": 85, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03021557000465691} +{"call": 86, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02895576017908752} +{"call": 87, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031830256804823875} +{"call": 88, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027472693007439375} +{"call": 89, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029829476960003376} +{"call": 90, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.837236261174659e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04893285594880581} +{"call": 91, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04032510402612388} +{"call": 92, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03115028399042785} +{"call": 93, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.036812482168897986} +{"call": 94, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027506768004968762} +{"call": 95, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028269713977351785} +{"call": 96, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02887068293057382} +{"call": 97, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029719524085521698} +{"call": 98, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027482945006340742} +{"call": 99, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027443549828603864} +{"call": 100, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.55330436982972e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04956601606681943} +{"call": 101, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030175888212397695} +{"call": 102, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028987753903493285} +{"call": 103, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027366556925699115} +{"call": 104, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029344687005504966} +{"call": 105, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03080619382672012} +{"call": 106, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027827709913253784} +{"call": 107, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03017034102231264} +{"call": 108, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032005571061745286} +{"call": 109, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02774458401836455} +{"call": 110, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 6.142548349615669e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04880955605767667} +{"call": 111, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02955208788625896} +{"call": 112, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028937323950231075} +{"call": 113, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027829610975459218} +{"call": 114, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02953988709487021} +{"call": 115, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792307292111218} +{"call": 116, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027460969053208828} +{"call": 117, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030527108116075397} +{"call": 118, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027340146945789456} +{"call": 119, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029006236931309104} +{"call": 120, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 6.350655365523561e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0489947278983891} +{"call": 121, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767459605820477} +{"call": 122, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029550391947850585} +{"call": 123, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02922022808343172} +{"call": 124, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028527398128062487} +{"call": 125, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028324957005679607} +{"call": 126, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02737982594408095} +{"call": 127, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0295121178496629} +{"call": 128, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02745826682075858} +{"call": 129, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027730507077649236} +{"call": 130, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.837886415346077e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.08338708290830255} +{"call": 131, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029432318871840835} +{"call": 132, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028584698913618922} +{"call": 133, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029232771135866642} +{"call": 134, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0319187140557915} +{"call": 135, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027561440831050277} +{"call": 136, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027588550001382828} +{"call": 137, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030140422051772475} +{"call": 138, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027381086023524404} +{"call": 139, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0274757940787822} +{"call": 140, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 6.386522238787783e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04864168097265065} +{"call": 141, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027741363970562816} +{"call": 142, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027275245869532228} +{"call": 143, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0336214960552752} +{"call": 144, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027451908215880394} +{"call": 145, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027885636081919074} +{"call": 146, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029159015975892544} +{"call": 147, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027737056137993932} +{"call": 148, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03190614003688097} +{"call": 149, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029823759105056524} +{"call": 150, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.757285492362209e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.050392891047522426} +{"call": 151, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03346710395999253} +{"call": 152, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031022086972370744} +{"call": 153, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027318830834701657} +{"call": 154, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.055052118841558695} +{"call": 155, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028761206893250346} +{"call": 156, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03238320699892938} +{"call": 157, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027931177988648415} +{"call": 158, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03169593820348382} +{"call": 159, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027365293819457293} +{"call": 160, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 6.520254968346886e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049519765889272094} +{"call": 161, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02895166794769466} +{"call": 162, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02781802206300199} +{"call": 163, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02813815395347774} +{"call": 164, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027891068952158093} +{"call": 165, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032490991987288} +{"call": 166, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027331538032740355} +{"call": 167, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028485606890171766} +{"call": 168, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028500049840658903} +{"call": 169, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027777537936344743} +{"call": 170, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 6.257109967035135e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04845138802193105} +{"call": 171, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030553166987374425} +{"call": 172, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027757691219449043} +{"call": 173, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02851104410365224} +{"call": 174, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028312748996540904} +{"call": 175, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027560896007344127} +{"call": 176, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029368205927312374} +{"call": 177, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02807215810753405} +{"call": 178, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02800590405240655} +{"call": 179, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03091717604547739} +{"call": 180, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 7.42704249077439e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0485278929118067} +{"call": 181, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027561157010495663} +{"call": 182, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027470665983855724} +{"call": 183, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03091207193210721} +{"call": 184, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027611305937170982} +{"call": 185, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027963078813627362} +{"call": 186, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027589946053922176} +{"call": 187, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030779460910707712} +{"call": 188, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03132968698628247} +{"call": 189, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027649004012346268} +{"call": 190, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 7.779194820400112e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04713768418878317} +{"call": 191, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03036517510190606} +{"call": 192, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02731182100251317} +{"call": 193, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028388642007485032} +{"call": 194, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029889889992773533} +{"call": 195, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027868197998031974} +{"call": 196, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02759560290724039} +{"call": 197, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757159690372646} +{"call": 198, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.041824032086879015} +{"call": 199, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0284376738127321} +{"call": 200, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 8.168862265187791e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048927724128589034} +{"call": 201, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027877870947122574} +{"call": 202, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027329506119713187} +{"call": 203, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028433709871023893} +{"call": 204, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02949072909541428} +{"call": 205, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02745970501564443} +{"call": 206, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028483696980401874} +{"call": 207, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02760965214110911} +{"call": 208, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028729411074891686} +{"call": 209, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030888063833117485} +{"call": 210, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 8.324377551363657e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05156160402111709} +{"call": 211, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027724382933229208} +{"call": 212, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030780419940128922} +{"call": 213, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02749948506243527} +{"call": 214, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0301337749697268} +{"call": 215, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029513705987483263} +{"call": 216, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028601508121937513} +{"call": 217, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028767121024429798} +{"call": 218, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027468569111078978} +{"call": 219, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027917354833334684} +{"call": 220, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 9.011186238423326e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049350205110386014} +{"call": 221, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027734142960980535} +{"call": 222, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028662159107625484} +{"call": 223, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028464179951697588} +{"call": 224, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027725656982511282} +{"call": 225, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029037612956017256} +{"call": 226, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03136940090917051} +{"call": 227, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02750754216685891} +{"call": 228, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027676103869453073} +{"call": 229, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028427782002836466} +{"call": 230, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 7.648225173442598e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04898681095801294} +{"call": 231, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03082093712873757} +{"call": 232, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02865586499683559} +{"call": 233, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027465115999802947} +{"call": 234, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030940650962293148} +{"call": 235, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028917433926835656} +{"call": 236, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028623994905501604} +{"call": 237, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027570723090320826} +{"call": 238, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029189255088567734} +{"call": 239, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02980904304422438} +{"call": 240, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 9.229487222476543e-07, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.051960935117676854} +{"call": 241, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028989382088184357} +{"call": 242, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02791233896277845} +{"call": 243, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033222455997020006} +{"call": 244, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03118162718601525} +{"call": 245, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03053732798434794} +{"call": 246, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02940528094768524} +{"call": 247, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02840999001637101} +{"call": 248, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02756459405645728} +{"call": 249, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027881554095074534} +{"call": 250, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.018934930232744e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048935323022305965} +{"call": 251, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029736814089119434} +{"call": 252, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02763625606894493} +{"call": 253, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02899294183589518} +{"call": 254, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027858274057507515} +{"call": 255, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02759230579249561} +{"call": 256, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03185460506938398} +{"call": 257, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02794993412680924} +{"call": 258, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027805439895018935} +{"call": 259, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.13109804294072092} +{"call": 260, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.0388666395966755e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04918746999464929} +{"call": 261, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029501993907615542} +{"call": 262, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028854135191068053} +{"call": 263, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0278149270452559} +{"call": 264, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.2532997829839587} +{"call": 265, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.043066082056611776} +{"call": 266, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029184395913034678} +{"call": 267, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028212437173351645} +{"call": 268, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02862364798784256} +{"call": 269, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02816361398436129} +{"call": 270, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2010782932226083e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04911762196570635} +{"call": 271, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02881068200804293} +{"call": 272, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027567900018766522} +{"call": 273, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03023502789437771} +{"call": 274, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028726818040013313} +{"call": 275, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02751788916066289} +{"call": 276, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03036904288455844} +{"call": 277, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028501140186563134} +{"call": 278, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027779603842645884} +{"call": 279, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04125890997238457} +{"call": 280, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.136766561613355e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.3088249019347131} +{"call": 281, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030194588005542755} +{"call": 282, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03262289008125663} +{"call": 283, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029744246043264866} +{"call": 284, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02861963608302176} +{"call": 285, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02750936383381486} +{"call": 286, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029726837063208222} +{"call": 287, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028483422938734293} +{"call": 288, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02744515798985958} +{"call": 289, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029452729038894176} +{"call": 290, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.131205208246288e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04836893710307777} +{"call": 291, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03145490004681051} +{"call": 292, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027599418070167303} +{"call": 293, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027711916947737336} +{"call": 294, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031009216094389558} +{"call": 295, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02759923110716045} +{"call": 296, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027983481995761395} +{"call": 297, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02891932800412178} +{"call": 298, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02759119914844632} +{"call": 299, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028599021956324577} +{"call": 300, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.057634972509694e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0483137231785804} +{"call": 301, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03637796710245311} +{"call": 302, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028556894045323133} +{"call": 303, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02827482298016548} +{"call": 304, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029081552987918258} +{"call": 305, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029240320902317762} +{"call": 306, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029805900994688272} +{"call": 307, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027696446049958467} +{"call": 308, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027916854014620185} +{"call": 309, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03208565292879939} +{"call": 310, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1296802470350936e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04849580186419189} +{"call": 311, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028475401923060417} +{"call": 312, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027579466812312603} +{"call": 313, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03079461003653705} +{"call": 314, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029562446055933833} +{"call": 315, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027680156053975224} +{"call": 316, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027658028062433004} +{"call": 317, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03269787901081145} +{"call": 318, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027458488009870052} +{"call": 319, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028569274814799428} +{"call": 320, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2282251514888911e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04848842695355415} +{"call": 321, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787300618365407} +{"call": 322, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028060826007276773} +{"call": 323, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02995365997776389} +{"call": 324, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0280387329403311} +{"call": 325, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030867684865370393} +{"call": 326, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02774846786633134} +{"call": 327, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03069221507757902} +{"call": 328, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027869760990142822} +{"call": 329, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027613066835328937} +{"call": 330, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.06428962311851e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04927585297264159} +{"call": 331, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03048542491160333} +{"call": 332, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031162540894001722} +{"call": 333, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027654941892251372} +{"call": 334, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027559122070670128} +{"call": 335, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030011883936822414} +{"call": 336, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02783049806021154} +{"call": 337, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02769338502548635} +{"call": 338, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03320040903054178} +{"call": 339, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028644982958212495} +{"call": 340, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2734457767304365e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047340727876871824} +{"call": 341, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03156863618642092} +{"call": 342, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027577752014622092} +{"call": 343, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028630201937630773} +{"call": 344, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029454985167831182} +{"call": 345, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775240200571716} +{"call": 346, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02966759796254337} +{"call": 347, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02954787597991526} +{"call": 348, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027701606042683125} +{"call": 349, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03348150197416544} +{"call": 350, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.317294708064172e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049421994015574455} +{"call": 351, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028111994033679366} +{"call": 352, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775761717930436} +{"call": 353, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029048939934000373} +{"call": 354, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03231405094265938} +{"call": 355, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027687371941283345} +{"call": 356, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03061500797048211} +{"call": 357, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772668399848044} +{"call": 358, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027704027947038412} +{"call": 359, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030170084908604622} +{"call": 360, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1794482796637522e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049151327926665545} +{"call": 361, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028273425064980984} +{"call": 362, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03149113594554365} +{"call": 363, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02788017108105123} +{"call": 364, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0275066199246794} +{"call": 365, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0288992659188807} +{"call": 366, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027517129899933934} +{"call": 367, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03153148083947599} +{"call": 368, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029605505988001823} +{"call": 369, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027737251948565245} +{"call": 370, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1298693032832746e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049626040970906615} +{"call": 371, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03391747805289924} +{"call": 372, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027614529943093657} +{"call": 373, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03388489410281181} +{"call": 374, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05199122382327914} +{"call": 375, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770390990190208} +{"call": 376, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029552668100222945} +{"call": 377, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02795276395045221} +{"call": 378, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03003249503672123} +{"call": 379, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030007408000528812} +{"call": 380, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1170165599400409e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04883576394058764} +{"call": 381, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031451602932065725} +{"call": 382, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027782385936006904} +{"call": 383, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03122498979791999} +{"call": 384, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767291897907853} +{"call": 385, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772806608118117} +{"call": 386, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03163115098141134} +{"call": 387, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030043564969673753} +{"call": 388, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030801388900727034} +{"call": 389, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028737597865983844} +{"call": 390, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1799426639427987e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04933387599885464} +{"call": 391, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033187619876116514} +{"call": 392, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02884262497536838} +{"call": 393, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02916985796764493} +{"call": 394, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030732508981600404} +{"call": 395, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029179539997130632} +{"call": 396, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028453487902879715} +{"call": 397, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03349834098480642} +{"call": 398, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03202860103920102} +{"call": 399, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03022672305814922} +{"call": 400, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1663024464969101e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05034038098528981} +{"call": 401, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.034892048221081495} +{"call": 402, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028916987124830484} +{"call": 403, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033831642009317875} +{"call": 404, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028022425016388297} +{"call": 405, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027984175132587552} +{"call": 406, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770313690416515} +{"call": 407, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028286811895668507} +{"call": 408, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029280941002070904} +{"call": 409, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029945305082947016} +{"call": 410, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1228511595856581e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04568295204080641} +{"call": 411, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03003839193843305} +{"call": 412, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02860393701121211} +{"call": 413, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027942727087065578} +{"call": 414, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027628595009446144} +{"call": 415, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02846901211887598} +{"call": 416, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027890998870134354} +{"call": 417, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028702490031719208} +{"call": 418, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028144839918240905} +{"call": 419, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027537277899682522} +{"call": 420, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.076453516155373e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047608583932742476} +{"call": 421, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.034590885043144226} +{"call": 422, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027716002892702818} +{"call": 423, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029064554953947663} +{"call": 424, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029534134082496166} +{"call": 425, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027659667190164328} +{"call": 426, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02771357912570238} +{"call": 427, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030869814101606607} +{"call": 428, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027703993022441864} +{"call": 429, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.042159599950537086} +{"call": 430, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.0856357788423645e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.050599152920767665} +{"call": 431, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03026259201578796} +{"call": 432, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029694878961890936} +{"call": 433, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02783372695557773} +{"call": 434, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031036359956488013} +{"call": 435, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029509969986975193} +{"call": 436, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767978608608246} +{"call": 437, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844727970659733} +{"call": 438, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031200450845062733} +{"call": 439, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027813317952677608} +{"call": 440, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1418890486899656e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.052368224831297994} +{"call": 441, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0296327390242368} +{"call": 442, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0284418067894876} +{"call": 443, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032647592946887016} +{"call": 444, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02762520406395197} +{"call": 445, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03279353701509535} +{"call": 446, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.035102207912132144} +{"call": 447, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031228143023326993} +{"call": 448, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029289240017533302} +{"call": 449, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028381878044456244} +{"call": 450, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.212405067424362e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05119464499875903} +{"call": 451, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029458811040967703} +{"call": 452, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029906100127846003} +{"call": 453, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027811290929093957} +{"call": 454, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028927481966093183} +{"call": 455, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029497601091861725} +{"call": 456, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027994249016046524} +{"call": 457, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02888393891043961} +{"call": 458, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02860243897885084} +{"call": 459, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.035007834900170565} +{"call": 460, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2427554281349276e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0517711800057441} +{"call": 461, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027572779916226864} +{"call": 462, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029460262041538954} +{"call": 463, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030334975104779005} +{"call": 464, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02758346381597221} +{"call": 465, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03169891703873873} +{"call": 466, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027838173089548945} +{"call": 467, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032653503119945526} +{"call": 468, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04022034490481019} +{"call": 469, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027608149219304323} +{"call": 470, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.0933576199294093e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04845528304576874} +{"call": 471, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02771294303238392} +{"call": 472, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029013148974627256} +{"call": 473, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028005647007375956} +{"call": 474, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030394895002245903} +{"call": 475, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028834212105721235} +{"call": 476, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027644420973956585} +{"call": 477, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027512490982189775} +{"call": 478, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029097181046381593} +{"call": 479, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757723513059318} +{"call": 480, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.0962322376456855e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04939213488250971} +{"call": 481, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030840965919196606} +{"call": 482, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03209409792907536} +{"call": 483, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027590295067057014} +{"call": 484, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775399386882782} +{"call": 485, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027789870975539088} +{"call": 486, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027699730126187205} +{"call": 487, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031040718778967857} +{"call": 488, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028103695949539542} +{"call": 489, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028711970197036862} +{"call": 490, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.244122477921474e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0486659228336066} +{"call": 491, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027757392963394523} +{"call": 492, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027699331985786557} +{"call": 493, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03178952308371663} +{"call": 494, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027565824100747705} +{"call": 495, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027638704981654882} +{"call": 496, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029815029120072722} +{"call": 497, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029439133126288652} +{"call": 498, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.034653049893677235} +{"call": 499, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04510398395359516} +{"call": 500, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2494342895917436e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04882283206097782} +{"call": 501, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03123435890302062} +{"call": 502, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027633502148091793} +{"call": 503, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767330687493086} +{"call": 504, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02910966705530882} +{"call": 505, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03125201887451112} +{"call": 506, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028662740951403975} +{"call": 507, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028258732054382563} +{"call": 508, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.046532809967175126} +{"call": 509, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027571904938668013} +{"call": 510, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.1945782751286673e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04934662999585271} +{"call": 511, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027561182156205177} +{"call": 512, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02914468594826758} +{"call": 513, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027941575041040778} +{"call": 514, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027726137079298496} +{"call": 515, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03339845407754183} +{"call": 516, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03393292799592018} +{"call": 517, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027517094975337386} +{"call": 518, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028311018832027912} +{"call": 519, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03215368581004441} +{"call": 520, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2197104739937415e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04894889495335519} +{"call": 521, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027836143970489502} +{"call": 522, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032011265866458416} +{"call": 523, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028613853035494685} +{"call": 524, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.052367095137014985} +{"call": 525, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0275813820771873} +{"call": 526, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03128750203177333} +{"call": 527, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029702586820349097} +{"call": 528, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279036199208349} +{"call": 529, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02970354096032679} +{"call": 530, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.3285400503076253e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04923057905398309} +{"call": 531, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029882929986342788} +{"call": 532, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02784700901247561} +{"call": 533, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02863707416690886} +{"call": 534, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03290931507945061} +{"call": 535, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027751533081755042} +{"call": 536, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027910812059417367} +{"call": 537, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03060645004734397} +{"call": 538, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027796634938567877} +{"call": 539, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0881015791092068} +{"call": 540, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.333399620074227e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04960336000658572} +{"call": 541, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027551291976124048} +{"call": 542, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029628689866513014} +{"call": 543, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02790537104010582} +{"call": 544, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0296600719448179} +{"call": 545, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02783731697127223} +{"call": 546, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027702133171260357} +{"call": 547, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02912793098948896} +{"call": 548, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02963927504606545} +{"call": 549, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030470489989966154} +{"call": 550, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.3725097468337614e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04867698298767209} +{"call": 551, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03156798589043319} +{"call": 552, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028251291951164603} +{"call": 553, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027609398122876883} +{"call": 554, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027411065995693207} +{"call": 555, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029477379051968455} +{"call": 556, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02963837399147451} +{"call": 557, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027712340001016855} +{"call": 558, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02745324303396046} +{"call": 559, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027575364103540778} +{"call": 560, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.4233404247347163e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04896900011226535} +{"call": 561, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 1.1361410540994257} +{"call": 562, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02778091700747609} +{"call": 563, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027952815173193812} +{"call": 564, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027644973946735263} +{"call": 565, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032789797987788916} +{"call": 566, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02738261898048222} +{"call": 567, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027564262971282005} +{"call": 568, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03226390900090337} +{"call": 569, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02858076710253954} +{"call": 570, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.2970798637004136e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04929455812089145} +{"call": 571, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029706917935982347} +{"call": 572, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775821601971984} +{"call": 573, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02969974698498845} +{"call": 574, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 1.2822124371305108} +{"call": 575, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02903408301062882} +{"call": 576, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031033519189804792} +{"call": 577, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792905503883958} +{"call": 578, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027953741140663624} +{"call": 579, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767345099709928} +{"call": 580, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.3637504953913877e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.050373297883197665} +{"call": 581, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031131956027820706} +{"call": 582, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031778899021446705} +{"call": 583, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02743327501229942} +{"call": 584, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029470288893207908} +{"call": 585, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031178503995761275} +{"call": 586, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03946553310379386} +{"call": 587, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02970266994088888} +{"call": 588, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02762943902052939} +{"call": 589, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02740889391861856} +{"call": 590, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.3657699367894477e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05005021602846682} +{"call": 591, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027621658984571695} +{"call": 592, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027418058831244707} +{"call": 593, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02977994387038052} +{"call": 594, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027881958056241274} +{"call": 595, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030835519079118967} +{"call": 596, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028476674109697342} +{"call": 597, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02875525807030499} +{"call": 598, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02754087490029633} +{"call": 599, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033875033957883716} +{"call": 600, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.3877069442325195e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04983936017379165} +{"call": 601, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03063389495946467} +{"call": 602, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027517618145793676} +{"call": 603, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02783395117148757} +{"call": 604, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03340905113145709} +{"call": 605, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027513801818713546} +{"call": 606, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027526818914338946} +{"call": 607, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02759595587849617} +{"call": 608, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0330125349573791} +{"call": 609, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844399912282825} +{"call": 610, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.3977691085913113e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04691551183350384} +{"call": 611, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027886514784768224} +{"call": 612, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02796169207431376} +{"call": 613, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03796628909185529} +{"call": 614, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028847900917753577} +{"call": 615, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02748700580559671} +{"call": 616, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032790558878332376} +{"call": 617, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02760706818662584} +{"call": 618, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02805011789314449} +{"call": 619, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03160797501914203} +{"call": 620, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.469717008324847e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.053060008911415935} +{"call": 621, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027489020954817533} +{"call": 622, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03059410909190774} +{"call": 623, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027794803027063608} +{"call": 624, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027733993949368596} +{"call": 625, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031169492984190583} +{"call": 626, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027819427894428372} +{"call": 627, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029780792072415352} +{"call": 628, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03463713498786092} +{"call": 629, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027919262181967497} +{"call": 630, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.4771216482047062e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04901804495602846} +{"call": 631, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03085930598899722} +{"call": 632, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028639932861551642} +{"call": 633, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03439997392706573} +{"call": 634, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03271620185114443} +{"call": 635, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029021277092397213} +{"call": 636, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029153048060834408} +{"call": 637, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029533250955864787} +{"call": 638, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0280142470728606} +{"call": 639, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02849332895129919} +{"call": 640, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.6183314745172257e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0480333820451051} +{"call": 641, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03754478390328586} +{"call": 642, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027831890154629946} +{"call": 643, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028040650067850947} +{"call": 644, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792932093143463} +{"call": 645, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02855852502398193} +{"call": 646, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028210554039105773} +{"call": 647, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028799151070415974} +{"call": 648, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027497804956510663} +{"call": 649, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02881360612809658} +{"call": 650, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.6514691202703634e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047181997913867235} +{"call": 651, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027603155933320522} +{"call": 652, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03207258111797273} +{"call": 653, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03006981499493122} +{"call": 654, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02764969994314015} +{"call": 655, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027939957100898027} +{"call": 656, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032847731141373515} +{"call": 657, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757523488253355} +{"call": 658, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027971958043053746} +{"call": 659, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028743432834744453} +{"call": 660, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.736831012734987e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0496632878202945} +{"call": 661, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028731059981510043} +{"call": 662, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02928225207142532} +{"call": 663, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028792785946279764} +{"call": 664, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029974092030897737} +{"call": 665, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029109155992046} +{"call": 666, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792072296142578} +{"call": 667, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027924710884690285} +{"call": 668, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0288415749091655} +{"call": 669, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02793709491379559} +{"call": 670, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.834269839031266e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04965979908592999} +{"call": 671, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030899297911673784} +{"call": 672, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030694488203153014} +{"call": 673, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792349411174655} +{"call": 674, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028338110074400902} +{"call": 675, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027716455049812794} +{"call": 676, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027875402010977268} +{"call": 677, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029267329955473542} +{"call": 678, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030221770983189344} +{"call": 679, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027655536076053977} +{"call": 680, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9485520205897737e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04886407405138016} +{"call": 681, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029211255023255944} +{"call": 682, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02984195202589035} +{"call": 683, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02839689888060093} +{"call": 684, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02945634094066918} +{"call": 685, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029913449892774224} +{"call": 686, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03159486991353333} +{"call": 687, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027809198014438152} +{"call": 688, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03202179120853543} +{"call": 689, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03218487906269729} +{"call": 690, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.953586404498031e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04930373909883201} +{"call": 691, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03214719891548157} +{"call": 692, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0277832739520818} +{"call": 693, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028153897961601615} +{"call": 694, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030155995162203908} +{"call": 695, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03171078092418611} +{"call": 696, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029745080042630434} +{"call": 697, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0282466528005898} +{"call": 698, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027489718981087208} +{"call": 699, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029685470974072814} +{"call": 700, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.908123602604854e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048467217944562435} +{"call": 701, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02739221998490393} +{"call": 702, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02788707404397428} +{"call": 703, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03150316816754639} +{"call": 704, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029302751179784536} +{"call": 705, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027891218895092607} +{"call": 706, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03718441189266741} +{"call": 707, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028783873887732625} +{"call": 708, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0275443431455642} +{"call": 709, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027700105914846063} +{"call": 710, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.8986629993180302e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048201176803559065} +{"call": 711, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03095009899698198} +{"call": 712, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028670110972598195} +{"call": 713, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027547423960641026} +{"call": 714, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.035801349906250834} +{"call": 715, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027722720056772232} +{"call": 716, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03177972696721554} +{"call": 717, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027563221054151654} +{"call": 718, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03183371899649501} +{"call": 719, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027823392068967223} +{"call": 720, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.8442464274759324e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0488520348444581} +{"call": 721, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02952042198739946} +{"call": 722, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027875381987541914} +{"call": 723, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027680212864652276} +{"call": 724, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027699015801772475} +{"call": 725, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028513118159025908} +{"call": 726, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028027598047629} +{"call": 727, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032058054115623236} +{"call": 728, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0276652870234102} +{"call": 729, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02999689686112106} +{"call": 730, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.8230125511036786e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048970636911690235} +{"call": 731, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027616608887910843} +{"call": 732, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02773899002932012} +{"call": 733, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030120714101940393} +{"call": 734, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03198875510133803} +{"call": 735, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031704423017799854} +{"call": 736, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027777596144005656} +{"call": 737, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031104271998628974} +{"call": 738, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02918735216371715} +{"call": 739, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028448802884668112} +{"call": 740, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9057561459015654e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05082888109609485} +{"call": 741, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02786582219414413} +{"call": 742, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027961881132796407} +{"call": 743, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029820358846336603} +{"call": 744, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0334139009937644} +{"call": 745, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028020408935844898} +{"call": 746, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028035733150318265} +{"call": 747, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0330861231777817} +{"call": 748, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03198864683508873} +{"call": 749, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029122679959982634} +{"call": 750, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.943648155967955e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05377853591926396} +{"call": 751, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05257281311787665} +{"call": 752, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031600612215697765} +{"call": 753, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03246566001325846} +{"call": 754, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027938984101638198} +{"call": 755, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028383811935782433} +{"call": 756, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030767050106078386} +{"call": 757, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03246187698096037} +{"call": 758, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029137765988707542} +{"call": 759, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027595350984483957} +{"call": 760, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.989659498797897e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05030520004220307} +{"call": 761, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02950378297828138} +{"call": 762, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027509510051459074} +{"call": 763, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03373205219395459} +{"call": 764, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029228515923023224} +{"call": 765, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030498848063871264} +{"call": 766, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03141477284952998} +{"call": 767, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028294231975451112} +{"call": 768, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028243115870282054} +{"call": 769, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02971015707589686} +{"call": 770, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9395242213033537e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04890789603814483} +{"call": 771, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028013493167236447} +{"call": 772, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033556590089574456} +{"call": 773, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02763607888482511} +{"call": 774, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027782213874161243} +{"call": 775, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027530438033863902} +{"call": 776, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030785467009991407} +{"call": 777, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032316778087988496} +{"call": 778, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029092062963172793} +{"call": 779, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02960299700498581} +{"call": 780, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9309364523109976e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04888196801766753} +{"call": 781, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029294820968061686} +{"call": 782, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029874423053115606} +{"call": 783, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027976342011243105} +{"call": 784, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03092727204784751} +{"call": 785, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027843437856063247} +{"call": 786, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028087031096220016} +{"call": 787, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033587204990908504} +{"call": 788, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027726616011932492} +{"call": 789, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027562255039811134} +{"call": 790, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.1622569849686316e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04736349708400667} +{"call": 791, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027654577046632767} +{"call": 792, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027548908023163676} +{"call": 793, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0331545399967581} +{"call": 794, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027801210060715675} +{"call": 795, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027969090035185218} +{"call": 796, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030966951046139002} +{"call": 797, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027686126995831728} +{"call": 798, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027953993063420057} +{"call": 799, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03232870204374194} +{"call": 800, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.1194436040999573e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04884151997976005} +{"call": 801, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033679113956168294} +{"call": 802, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028594925068318844} +{"call": 803, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030172242084518075} +{"call": 804, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02837907406501472} +{"call": 805, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03146753506734967} +{"call": 806, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028397932881489396} +{"call": 807, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03659625514410436} +{"call": 808, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.3197854240424931} +{"call": 809, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027765382081270218} +{"call": 810, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.115557436439516e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0490929689258337} +{"call": 811, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030179580906406045} +{"call": 812, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027821428840979934} +{"call": 813, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029688318027183414} +{"call": 814, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028771609999239445} +{"call": 815, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029148055939003825} +{"call": 816, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03333915397524834} +{"call": 817, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03260284988209605} +{"call": 818, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027583749033510685} +{"call": 819, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029130219016224146} +{"call": 820, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.131918045771305e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049057506024837494} +{"call": 821, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03088376601226628} +{"call": 822, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02763308398425579} +{"call": 823, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02793226297944784} +{"call": 824, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0323835089802742} +{"call": 825, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027932920027524233} +{"call": 826, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027796439826488495} +{"call": 827, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03018070082180202} +{"call": 828, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767558000050485} +{"call": 829, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027735852170735598} +{"call": 830, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.216637064119187e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04805497778579593} +{"call": 831, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03323098900727928} +{"call": 832, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02777024800889194} +{"call": 833, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030718248104676604} +{"call": 834, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02785269496962428} +{"call": 835, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030893479008227587} +{"call": 836, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029350760160014033} +{"call": 837, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030816761078312993} +{"call": 838, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027624286012724042} +{"call": 839, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028234953060746193} +{"call": 840, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.1689252414900853e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048541516065597534} +{"call": 841, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0310461709741503} +{"call": 842, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027713262010365725} +{"call": 843, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028516157064586878} +{"call": 844, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03291624900884926} +{"call": 845, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02744070696644485} +{"call": 846, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028835629113018513} +{"call": 847, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028664507204666734} +{"call": 848, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02749522589147091} +{"call": 849, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031049192883074284} +{"call": 850, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.207175277734718e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04890782595612109} +{"call": 851, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029059438966214657} +{"call": 852, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02851984091103077} +{"call": 853, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031057005049660802} +{"call": 854, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027862378861755133} +{"call": 855, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028429646976292133} +{"call": 856, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030062994919717312} +{"call": 857, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02768080704845488} +{"call": 858, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.06045904802158475} +{"call": 859, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02939013601280749} +{"call": 860, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.1628150549281982e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049137885915115476} +{"call": 861, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027995181968435645} +{"call": 862, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027920346008613706} +{"call": 863, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03010767395608127} +{"call": 864, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028800785075873137} +{"call": 865, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027890828903764486} +{"call": 866, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027821936877444386} +{"call": 867, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03072905191220343} +{"call": 868, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029288702877238393} +{"call": 869, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029085923917591572} +{"call": 870, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.161814024767334e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0484663569368422} +{"call": 871, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03202108992263675} +{"call": 872, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03233197913505137} +{"call": 873, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028938295086845756} +{"call": 874, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03166475798934698} +{"call": 875, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02782901795580983} +{"call": 876, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03508946392685175} +{"call": 877, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030529623851180077} +{"call": 878, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027728711953386664} +{"call": 879, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027817040914669633} +{"call": 880, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.058965305924832e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04868929390795529} +{"call": 881, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030392687069252133} +{"call": 882, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027820757823064923} +{"call": 883, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789392787963152} +{"call": 884, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02795798797160387} +{"call": 885, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027939270017668605} +{"call": 886, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028294597985222936} +{"call": 887, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05504376790486276} +{"call": 888, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02900231396779418} +{"call": 889, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028091955929994583} +{"call": 890, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.083662316039371e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04885273310355842} +{"call": 891, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028878699988126755} +{"call": 892, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02810229011811316} +{"call": 893, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028748401906341314} +{"call": 894, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027757368981838226} +{"call": 895, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02975527592934668} +{"call": 896, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029007131000980735} +{"call": 897, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030633767135441303} +{"call": 898, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02786101889796555} +{"call": 899, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03015453997068107} +{"call": 900, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9976670213388543e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04935821401886642} +{"call": 901, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027989672031253576} +{"call": 902, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027777563082054257} +{"call": 903, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02751552197150886} +{"call": 904, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032647441141307354} +{"call": 905, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.17465561674907804} +{"call": 906, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02777634421363473} +{"call": 907, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02900324435904622} +{"call": 908, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027474678121507168} +{"call": 909, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02951127989217639} +{"call": 910, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9396219160809585e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04918672097846866} +{"call": 911, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031472515780478716} +{"call": 912, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.10423787403851748} +{"call": 913, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029827957041561604} +{"call": 914, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028080454096198082} +{"call": 915, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02972274972125888} +{"call": 916, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279288231395185} +{"call": 917, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029616624116897583} +{"call": 918, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03027309011667967} +{"call": 919, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027728511951863766} +{"call": 920, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 1.9363816003890113e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04891455080360174} +{"call": 921, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027921037748456} +{"call": 922, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02761785825714469} +{"call": 923, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030049722641706467} +{"call": 924, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029605844989418983} +{"call": 925, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027274616062641144} +{"call": 926, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03000700194388628} +{"call": 927, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033286591060459614} +{"call": 928, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027637725695967674} +{"call": 929, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029061105102300644} +{"call": 930, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.016051271437609e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049716990906745195} +{"call": 931, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.3192648231051862} +{"call": 932, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029323061928153038} +{"call": 933, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02790468605235219} +{"call": 934, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0305430400185287} +{"call": 935, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027835679706186056} +{"call": 936, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02841872302815318} +{"call": 937, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.035866458900272846} +{"call": 938, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028372690081596375} +{"call": 939, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028811792843043804} +{"call": 940, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.0906406004468487e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05458871042355895} +{"call": 941, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027579624205827713} +{"call": 942, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027845091186463833} +{"call": 943, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028417701832950115} +{"call": 944, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028574618976563215} +{"call": 945, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028290680143982172} +{"call": 946, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028287113178521395} +{"call": 947, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029794906731694937} +{"call": 948, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02825667103752494} +{"call": 949, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028599260840564966} +{"call": 950, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.0529297253321013e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05055396491661668} +{"call": 951, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05584488483145833} +{"call": 952, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03151311678811908} +{"call": 953, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027840265072882175} +{"call": 954, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029378070030361414} +{"call": 955, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03159330505877733} +{"call": 956, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02758670598268509} +{"call": 957, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03138229623436928} +{"call": 958, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028405196964740753} +{"call": 959, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031109235249459743} +{"call": 960, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.0891312832318563e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04914957331493497} +{"call": 961, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027912139892578125} +{"call": 962, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02778742415830493} +{"call": 963, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02818915667012334} +{"call": 964, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0276515269652009} +{"call": 965, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027826722711324692} +{"call": 966, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027759586926549673} +{"call": 967, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027790294028818607} +{"call": 968, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031964011024683714} +{"call": 969, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027970157098025084} +{"call": 970, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.1893488023401965e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048486588057130575} +{"call": 971, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027553386986255646} +{"call": 972, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03172840876504779} +{"call": 973, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03673138376325369} +{"call": 974, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02760070003569126} +{"call": 975, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027776373084634542} +{"call": 976, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028914521913975477} +{"call": 977, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03039258299395442} +{"call": 978, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028080997057259083} +{"call": 979, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028088591992855072} +{"call": 980, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.21171811864161e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05074041010811925} +{"call": 981, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029776104725897312} +{"call": 982, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030180647037923336} +{"call": 983, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027425740845501423} +{"call": 984, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029141978360712528} +{"call": 985, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029541753698140383} +{"call": 986, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02768132509663701} +{"call": 987, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02867206698283553} +{"call": 988, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031510679982602596} +{"call": 989, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.038851025979965925} +{"call": 990, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.180670513627199e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04895076667889953} +{"call": 991, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02758315298706293} +{"call": 992, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030261820182204247} +{"call": 993, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028090002946555614} +{"call": 994, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02755207335576415} +{"call": 995, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03562887804582715} +{"call": 996, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02881589299067855} +{"call": 997, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027769902255386114} +{"call": 998, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027679810766130686} +{"call": 999, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027708583045750856} +{"call": 1000, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.2833651003900257e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048665997106581926} +{"call": 1001, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03105640970170498} +{"call": 1002, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027630310971289873} +{"call": 1003, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030217550229281187} +{"call": 1004, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03072108281776309} +{"call": 1005, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02740770997479558} +{"call": 1006, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029458108823746443} +{"call": 1007, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031656851060688496} +{"call": 1008, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02907941397279501} +{"call": 1009, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02874183002859354} +{"call": 1010, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.295025457915584e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048354119062423706} +{"call": 1011, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844805736094713} +{"call": 1012, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028454766608774662} +{"call": 1013, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0297753787599504} +{"call": 1014, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028281289618462324} +{"call": 1015, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028012927155941725} +{"call": 1016, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027970573864877224} +{"call": 1017, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03253326937556267} +{"call": 1018, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027917832136154175} +{"call": 1019, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029347478412091732} +{"call": 1020, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.2120538376378723e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05474317725747824} +{"call": 1021, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03240330517292023} +{"call": 1022, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02756640873849392} +{"call": 1023, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027628649957478046} +{"call": 1024, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029682690277695656} +{"call": 1025, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027848790865391493} +{"call": 1026, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02981374505907297} +{"call": 1027, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03063596924766898} +{"call": 1028, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787543088197708} +{"call": 1029, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02928466722369194} +{"call": 1030, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.406995579255346e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04823806928470731} +{"call": 1031, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02950685005635023} +{"call": 1032, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028401579707860947} +{"call": 1033, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029262538067996502} +{"call": 1034, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029417520854622126} +{"call": 1035, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027664069086313248} +{"call": 1036, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030424695927649736} +{"call": 1037, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02783822175115347} +{"call": 1038, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031583039090037346} +{"call": 1039, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030735358130186796} +{"call": 1040, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.413254591833158e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04892888991162181} +{"call": 1041, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030677803326398134} +{"call": 1042, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027695432305336} +{"call": 1043, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029523163102567196} +{"call": 1044, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03403380699455738} +{"call": 1045, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279376907274127} +{"call": 1046, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027477098163217306} +{"call": 1047, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032255472615361214} +{"call": 1048, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029367944225668907} +{"call": 1049, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02796107204630971} +{"call": 1050, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.414312833105381e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048745498061180115} +{"call": 1051, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0311612649820745} +{"call": 1052, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029565992299467325} +{"call": 1053, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02791293477639556} +{"call": 1054, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03170700557529926} +{"call": 1055, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844743337482214} +{"call": 1056, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028475796803832054} +{"call": 1057, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029152306262403727} +{"call": 1058, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027926506008952856} +{"call": 1059, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029019521083682775} +{"call": 1060, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.30531286908979e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047293879091739655} +{"call": 1061, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789399493485689} +{"call": 1062, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030824559275060892} +{"call": 1063, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029589036013931036} +{"call": 1064, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027984562795609236} +{"call": 1065, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03087271796539426} +{"call": 1066, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029564901255071163} +{"call": 1067, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029989332426339388} +{"call": 1068, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029702333267778158} +{"call": 1069, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027805821038782597} +{"call": 1070, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.5204176724406157e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04757847776636481} +{"call": 1071, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029077059123665094} +{"call": 1072, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030731898732483387} +{"call": 1073, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029621047899127007} +{"call": 1074, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02894709026440978} +{"call": 1075, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027961426880210638} +{"call": 1076, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032745236065238714} +{"call": 1077, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027798860799521208} +{"call": 1078, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02862227987498045} +{"call": 1079, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029265224002301693} +{"call": 1080, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.537517175561369e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04838648717850447} +{"call": 1081, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03360967990010977} +{"call": 1082, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757843676954508} +{"call": 1083, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027737505733966827} +{"call": 1084, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03366567799821496} +{"call": 1085, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027350088581442833} +{"call": 1086, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029464010149240494} +{"call": 1087, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030711160972714424} +{"call": 1088, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02856503799557686} +{"call": 1089, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028975007124245167} +{"call": 1090, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.5264276453469247e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04957194719463587} +{"call": 1091, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03129900200292468} +{"call": 1092, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030482932925224304} +{"call": 1093, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03019161568954587} +{"call": 1094, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02788644889369607} +{"call": 1095, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029695661272853613} +{"call": 1096, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767870808020234} +{"call": 1097, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027620553970336914} +{"call": 1098, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029858273919671774} +{"call": 1099, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027713904157280922} +{"call": 1100, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.5128141790777815e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049201956018805504} +{"call": 1101, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03515936899930239} +{"call": 1102, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027603301219642162} +{"call": 1103, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02762879617512226} +{"call": 1104, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031039464753121138} +{"call": 1105, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02759537799283862} +{"call": 1106, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02806313894689083} +{"call": 1107, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03495862986892462} +{"call": 1108, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027865047100931406} +{"call": 1109, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029052427038550377} +{"call": 1110, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.4933146153035026e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049290485214442015} +{"call": 1111, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0277071469463408} +{"call": 1112, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03065510280430317} +{"call": 1113, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027854626066982746} +{"call": 1114, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027709010057151318} +{"call": 1115, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027687407098710537} +{"call": 1116, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03024855675175786} +{"call": 1117, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028159985784441233} +{"call": 1118, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030606599058955908} +{"call": 1119, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030607759952545166} +{"call": 1120, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.290340667170458e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05031390022486448} +{"call": 1121, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03322096634656191} +{"call": 1122, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02784595498815179} +{"call": 1123, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0276699080131948} +{"call": 1124, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02879476035013795} +{"call": 1125, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02778328489512205} +{"call": 1126, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027701816987246275} +{"call": 1127, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029252152889966965} +{"call": 1128, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030741503927856684} +{"call": 1129, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030601411126554012} +{"call": 1130, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.3586479194258674e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047684586606919765} +{"call": 1131, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03538735583424568} +{"call": 1132, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027735366951674223} +{"call": 1133, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027831729967147112} +{"call": 1134, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0294333528727293} +{"call": 1135, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028036939911544323} +{"call": 1136, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0340770878829062} +{"call": 1137, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03099504904821515} +{"call": 1138, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027668936178088188} +{"call": 1139, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030154037289321423} +{"call": 1140, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.3090970941882165e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04923267615959048} +{"call": 1141, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027850821614265442} +{"call": 1142, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02774990489706397} +{"call": 1143, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030048851389437914} +{"call": 1144, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844855096191168} +{"call": 1145, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027945118956267834} +{"call": 1146, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030561324674636126} +{"call": 1147, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027711045928299427} +{"call": 1148, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029710300266742706} +{"call": 1149, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03103521978482604} +{"call": 1150, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.435834946669641e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048673662822693586} +{"call": 1151, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02765098586678505} +{"call": 1152, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03118894388899207} +{"call": 1153, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028320483397692442} +{"call": 1154, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029501231387257576} +{"call": 1155, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02823758404701948} +{"call": 1156, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027559508103877306} +{"call": 1157, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032441234681755304} +{"call": 1158, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02796964393928647} +{"call": 1159, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027629065793007612} +{"call": 1160, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.3837164236277655e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04933922691270709} +{"call": 1161, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027872466947883368} +{"call": 1162, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02788928523659706} +{"call": 1163, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031838564202189445} +{"call": 1164, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02776502538472414} +{"call": 1165, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05325461970642209} +{"call": 1166, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028687940910458565} +{"call": 1167, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.10290924599394202} +{"call": 1168, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028897385112941265} +{"call": 1169, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027982096187770367} +{"call": 1170, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.5067137363530733e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04947286797687411} +{"call": 1171, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028991966973990202} +{"call": 1172, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02988362591713667} +{"call": 1173, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03081613127142191} +{"call": 1174, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028682326897978783} +{"call": 1175, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027798313181847334} +{"call": 1176, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03268987499177456} +{"call": 1177, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028666961006820202} +{"call": 1178, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029132045339792967} +{"call": 1179, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027695504017174244} +{"call": 1180, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.5039481054047913e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048986976966261864} +{"call": 1181, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030601055826991796} +{"call": 1182, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027610841672867537} +{"call": 1183, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033123000990599394} +{"call": 1184, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030750149860978127} +{"call": 1185, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027891312260180712} +{"call": 1186, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029368812683969736} +{"call": 1187, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028417817782610655} +{"call": 1188, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0277259168215096} +{"call": 1189, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029121916741132736} +{"call": 1190, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.5832150281213883e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04860881110653281} +{"call": 1191, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027635097969323397} +{"call": 1192, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02830873103812337} +{"call": 1193, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030193305108696222} +{"call": 1194, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02889499394223094} +{"call": 1195, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027897482737898827} +{"call": 1196, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03148389793932438} +{"call": 1197, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772311493754387} +{"call": 1198, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027772755827754736} +{"call": 1199, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03268491989001632} +{"call": 1200, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.593508088852512e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04922058619558811} +{"call": 1201, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02825003396719694} +{"call": 1202, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027959576807916164} +{"call": 1203, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027967935893684626} +{"call": 1204, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027637032326310873} +{"call": 1205, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02965176897123456} +{"call": 1206, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027582278940826654} +{"call": 1207, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02984529035165906} +{"call": 1208, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028097938746213913} +{"call": 1209, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02793763903900981} +{"call": 1210, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.584472016489938e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049346532206982374} +{"call": 1211, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03341894783079624} +{"call": 1212, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033521002158522606} +{"call": 1213, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029670365154743195} +{"call": 1214, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028879432938992977} +{"call": 1215, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027795024681836367} +{"call": 1216, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027830203995108604} +{"call": 1217, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028015888296067715} +{"call": 1218, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031486282125115395} +{"call": 1219, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030171758960932493} +{"call": 1220, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.6730638955737156e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04904395807534456} +{"call": 1221, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03101749112829566} +{"call": 1222, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027790984138846397} +{"call": 1223, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028541676234453917} +{"call": 1224, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028990395832806826} +{"call": 1225, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02776125492528081} +{"call": 1226, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02801878470927477} +{"call": 1227, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027644475921988487} +{"call": 1228, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030475393868982792} +{"call": 1229, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027612709905952215} +{"call": 1230, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.6316213731168586e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04833054728806019} +{"call": 1231, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02818662580102682} +{"call": 1232, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030027400236576796} +{"call": 1233, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.034580593928694725} +{"call": 1234, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028167590964585543} +{"call": 1235, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02910058805719018} +{"call": 1236, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03002267610281706} +{"call": 1237, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03255607606843114} +{"call": 1238, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03234670916572213} +{"call": 1239, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03350505558773875} +{"call": 1240, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.520573036086823e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05096512194722891} +{"call": 1241, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029395044781267643} +{"call": 1242, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02928012888878584} +{"call": 1243, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027659195009618998} +{"call": 1244, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03220253111794591} +{"call": 1245, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02793527115136385} +{"call": 1246, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03227818105369806} +{"call": 1247, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03360191825777292} +{"call": 1248, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027943965047597885} +{"call": 1249, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028554765041917562} +{"call": 1250, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.707558460291819e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04828891111537814} +{"call": 1251, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02993318298831582} +{"call": 1252, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027724900748580694} +{"call": 1253, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028807215858250856} +{"call": 1254, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.3247164571657777} +{"call": 1255, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.09567301580682397} +{"call": 1256, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02922633895650506} +{"call": 1257, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03142202598974109} +{"call": 1258, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027596773114055395} +{"call": 1259, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029279320035129786} +{"call": 1260, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.721684484952421e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04848366789519787} +{"call": 1261, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027743434999138117} +{"call": 1262, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030031536240130663} +{"call": 1263, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027586617041379213} +{"call": 1264, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027606685645878315} +{"call": 1265, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029549870174378157} +{"call": 1266, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0330777489580214} +{"call": 1267, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028077625203877687} +{"call": 1268, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0307576940394938} +{"call": 1269, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02800353104248643} +{"call": 1270, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.769301455046998e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04836786026135087} +{"call": 1271, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0300136711448431} +{"call": 1272, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028442781884223223} +{"call": 1273, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772427909076214} +{"call": 1274, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030021790880709887} +{"call": 1275, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028897245414555073} +{"call": 1276, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02940341690555215} +{"call": 1277, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027894936967641115} +{"call": 1278, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028816196136176586} +{"call": 1279, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029914953745901585} +{"call": 1280, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.7320830614903503e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.1638917070813477} +{"call": 1281, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028315010014921427} +{"call": 1282, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028911476954817772} +{"call": 1283, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05335309915244579} +{"call": 1284, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028135831002146006} +{"call": 1285, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772604301571846} +{"call": 1286, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02836179593577981} +{"call": 1287, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033263769932091236} +{"call": 1288, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02934965491294861} +{"call": 1289, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027775070164352655} +{"call": 1290, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.7319660869210283e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.1092236009426415} +{"call": 1291, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799359615892172} +{"call": 1292, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02912013605237007} +{"call": 1293, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027711933944374323} +{"call": 1294, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029297387693077326} +{"call": 1295, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030469504185020924} +{"call": 1296, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027913252357393503} +{"call": 1297, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027999394107609987} +{"call": 1298, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032260789070278406} +{"call": 1299, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027895084116607904} +{"call": 1300, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.652846504872489e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05060544516891241} +{"call": 1301, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.08521171100437641} +{"call": 1302, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0977744972333312} +{"call": 1303, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02766210213303566} +{"call": 1304, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027681773994117975} +{"call": 1305, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02879755012691021} +{"call": 1306, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03153849579393864} +{"call": 1307, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02779678301885724} +{"call": 1308, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028222363907843828} +{"call": 1309, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02794166700914502} +{"call": 1310, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.658573855333175e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04943031584843993} +{"call": 1311, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02847183123230934} +{"call": 1312, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03515438316389918} +{"call": 1313, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027853087056428194} +{"call": 1314, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028576374985277653} +{"call": 1315, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031321580056101084} +{"call": 1316, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02785751922056079} +{"call": 1317, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02882356196641922} +{"call": 1318, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027749919798225164} +{"call": 1319, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027885590214282274} +{"call": 1320, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.7157570663420944e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05251329764723778} +{"call": 1321, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02942764712497592} +{"call": 1322, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02732176799327135} +{"call": 1323, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.20308378105983138} +{"call": 1324, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02786135021597147} +{"call": 1325, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770898211747408} +{"call": 1326, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029946499038487673} +{"call": 1327, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02994753187522292} +{"call": 1328, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030879803001880646} +{"call": 1329, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027882463298738003} +{"call": 1330, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.8729220019981743e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04916383605450392} +{"call": 1331, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030640430748462677} +{"call": 1332, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027898869011551142} +{"call": 1333, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027719099074602127} +{"call": 1334, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029309678822755814} +{"call": 1335, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027837171219289303} +{"call": 1336, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028041900135576725} +{"call": 1337, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0328653110191226} +{"call": 1338, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032113876193761826} +{"call": 1339, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02781366603448987} +{"call": 1340, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.8831586826783354e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04836890008300543} +{"call": 1341, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770281909033656} +{"call": 1342, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029941869899630547} +{"call": 1343, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03643183782696724} +{"call": 1344, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027707851957529783} +{"call": 1345, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03341174777597189} +{"call": 1346, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029381813947111368} +{"call": 1347, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029184444807469845} +{"call": 1348, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027612056117504835} +{"call": 1349, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030496292747557163} +{"call": 1350, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.8942720966814215e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048178196884691715} +{"call": 1351, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02951148897409439} +{"call": 1352, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027714640833437443} +{"call": 1353, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027714095078408718} +{"call": 1354, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03182293428108096} +{"call": 1355, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02756298379972577} +{"call": 1356, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027843653224408627} +{"call": 1357, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029393962118774652} +{"call": 1358, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767136413604021} +{"call": 1359, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028557131066918373} +{"call": 1360, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.951585082531779e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05183291295543313} +{"call": 1361, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027991538867354393} +{"call": 1362, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02788329729810357} +{"call": 1363, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028123851865530014} +{"call": 1364, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028147248085588217} +{"call": 1365, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02769857970997691} +{"call": 1366, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799532562494278} +{"call": 1367, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03145174216479063} +{"call": 1368, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029338020831346512} +{"call": 1369, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027797553222626448} +{"call": 1370, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.970389148678898e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04995199479162693} +{"call": 1371, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027795241214334965} +{"call": 1372, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027925316710025072} +{"call": 1373, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03214439982548356} +{"call": 1374, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0321215121075511} +{"call": 1375, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03314246702939272} +{"call": 1376, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028510154224932194} +{"call": 1377, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02801336906850338} +{"call": 1378, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799239382147789} +{"call": 1379, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03235284611582756} +{"call": 1380, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.9914667244016356e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04820257890969515} +{"call": 1381, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027879189234226942} +{"call": 1382, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03213253803551197} +{"call": 1383, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027690012007951736} +{"call": 1384, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027790050953626633} +{"call": 1385, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029220879077911377} +{"call": 1386, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02779004629701376} +{"call": 1387, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.2569357817992568} +{"call": 1388, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030370612163096666} +{"call": 1389, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028881805948913097} +{"call": 1390, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.981142330891584e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04849139414727688} +{"call": 1391, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027639316860586405} +{"call": 1392, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027591366320848465} +{"call": 1393, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027920654974877834} +{"call": 1394, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033207214903086424} +{"call": 1395, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029124869965016842} +{"call": 1396, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027833789121359587} +{"call": 1397, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03226371621713042} +{"call": 1398, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02783870929852128} +{"call": 1399, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028863522689789534} +{"call": 1400, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.001943231063276e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04854855267331004} +{"call": 1401, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029527178034186363} +{"call": 1402, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028444261290133} +{"call": 1403, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030795915983617306} +{"call": 1404, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027664984110742807} +{"call": 1405, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030386343598365784} +{"call": 1406, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02814390929415822} +{"call": 1407, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031765079125761986} +{"call": 1408, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028826129157096148} +{"call": 1409, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027647463139146566} +{"call": 1410, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.8799069930093563e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049101178999990225} +{"call": 1411, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02906615799292922} +{"call": 1412, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029525541234761477} +{"call": 1413, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03112171497195959} +{"call": 1414, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02851621899753809} +{"call": 1415, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028045980725437403} +{"call": 1416, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02794886101037264} +{"call": 1417, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02993740001693368} +{"call": 1418, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02766295289620757} +{"call": 1419, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02961046015843749} +{"call": 1420, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.930285109815016e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04888512101024389} +{"call": 1421, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028485984075814486} +{"call": 1422, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028698657173663378} +{"call": 1423, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028088982217013836} +{"call": 1424, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028225697111338377} +{"call": 1425, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03120457474142313} +{"call": 1426, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033888794016093016} +{"call": 1427, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028136076871305704} +{"call": 1428, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02869669534265995} +{"call": 1429, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757569309324026} +{"call": 1430, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.9180649451028805e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.052563006058335304} +{"call": 1431, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028567108791321516} +{"call": 1432, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030187110882252455} +{"call": 1433, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028241652064025402} +{"call": 1434, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029157161246985197} +{"call": 1435, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029000672046095133} +{"call": 1436, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02758534299209714} +{"call": 1437, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027776308823376894} +{"call": 1438, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029321444686502218} +{"call": 1439, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02831097086891532} +{"call": 1440, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.923245983876374e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047838366124778986} +{"call": 1441, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02836922788992524} +{"call": 1442, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027865563984960318} +{"call": 1443, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027045409195125103} +{"call": 1444, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02814856916666031} +{"call": 1445, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02836420712992549} +{"call": 1446, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027721148915588856} +{"call": 1447, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02850846154615283} +{"call": 1448, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028092639055103064} +{"call": 1449, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03139309585094452} +{"call": 1450, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.920733433001989e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04886577092111111} +{"call": 1451, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027780150063335896} +{"call": 1452, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027821077965199947} +{"call": 1453, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02951085800305009} +{"call": 1454, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027942723128944635} +{"call": 1455, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028017633128911257} +{"call": 1456, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02984259696677327} +{"call": 1457, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02830726094543934} +{"call": 1458, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029934552032500505} +{"call": 1459, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027770265005528927} +{"call": 1460, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.9475558650980854e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05103993369266391} +{"call": 1461, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03233920317143202} +{"call": 1462, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027647939044982195} +{"call": 1463, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027638299856334925} +{"call": 1464, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02866838499903679} +{"call": 1465, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02974148141220212} +{"call": 1466, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028915942180901766} +{"call": 1467, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027810574043542147} +{"call": 1468, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028680745977908373} +{"call": 1469, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02953425981104374} +{"call": 1470, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.909053933839091e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048919811844825745} +{"call": 1471, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027842686977237463} +{"call": 1472, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792145498096943} +{"call": 1473, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0290071670897305} +{"call": 1474, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029224825091660023} +{"call": 1475, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030301930848509073} +{"call": 1476, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030432545114308596} +{"call": 1477, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027768719010055065} +{"call": 1478, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027684805914759636} +{"call": 1479, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032759588211774826} +{"call": 1480, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.9609799040507212e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04851911496371031} +{"call": 1481, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030620960984379053} +{"call": 1482, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030163124203681946} +{"call": 1483, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02820682292804122} +{"call": 1484, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027717091143131256} +{"call": 1485, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028422869741916656} +{"call": 1486, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02785447333008051} +{"call": 1487, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03259671665728092} +{"call": 1488, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03422664711251855} +{"call": 1489, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789084007963538} +{"call": 1490, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 2.988474353149302e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048555552028119564} +{"call": 1491, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.07986186910420656} +{"call": 1492, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02773834904655814} +{"call": 1493, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02813794557005167} +{"call": 1494, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.043295739218592644} +{"call": 1495, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027720268815755844} +{"call": 1496, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028963904827833176} +{"call": 1497, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028368688188493252} +{"call": 1498, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02859635604545474} +{"call": 1499, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027891751378774643} +{"call": 1500, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.1732738072256256e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049221171997487545} +{"call": 1501, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027783758006989956} +{"call": 1502, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03009368386119604} +{"call": 1503, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027409323025494814} +{"call": 1504, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028186592739075422} +{"call": 1505, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03136540483683348} +{"call": 1506, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028028320986777544} +{"call": 1507, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027904227375984192} +{"call": 1508, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03194285603240132} +{"call": 1509, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02795462589710951} +{"call": 1510, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.1816586179248673e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04880498768761754} +{"call": 1511, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028534431010484695} +{"call": 1512, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030447437893599272} +{"call": 1513, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02765644807368517} +{"call": 1514, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029327751137316227} +{"call": 1515, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029624201823025942} +{"call": 1516, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027938283048570156} +{"call": 1517, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03249889286234975} +{"call": 1518, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028864311054348946} +{"call": 1519, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02828336413949728} +{"call": 1520, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.237485921985451e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04921716591343284} +{"call": 1521, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028222890105098486} +{"call": 1522, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02786918729543686} +{"call": 1523, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028445797972381115} +{"call": 1524, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030614133924245834} +{"call": 1525, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028574281837791204} +{"call": 1526, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02779508801177144} +{"call": 1527, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03137347428128123} +{"call": 1528, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789704082533717} +{"call": 1529, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030941805336624384} +{"call": 1530, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.2771072910682296e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.052567292004823685} +{"call": 1531, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027995881158858538} +{"call": 1532, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027708888985216618} +{"call": 1533, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032238516956567764} +{"call": 1534, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0275085368193686} +{"call": 1535, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0323396697640419} +{"call": 1536, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029134029988199472} +{"call": 1537, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02812920929864049} +{"call": 1538, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030610573012381792} +{"call": 1539, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799333818256855} +{"call": 1540, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.336502237342974e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04786955192685127} +{"call": 1541, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028104387689381838} +{"call": 1542, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031049150973558426} +{"call": 1543, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028128518722951412} +{"call": 1544, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031169456895440817} +{"call": 1545, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0477143325842917} +{"call": 1546, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02849781420081854} +{"call": 1547, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.036731048952788115} +{"call": 1548, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02808697707951069} +{"call": 1549, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027740854304283857} +{"call": 1550, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.39022230841054e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04895409382879734} +{"call": 1551, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029172293841838837} +{"call": 1552, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757956925779581} +{"call": 1553, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027866042219102383} +{"call": 1554, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027961810119450092} +{"call": 1555, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033949810080230236} +{"call": 1556, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0278588579967618} +{"call": 1557, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.050709500908851624} +{"call": 1558, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033569721039384604} +{"call": 1559, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028022451791912317} +{"call": 1560, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.394205659316484e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049515146762132645} +{"call": 1561, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030635681934654713} +{"call": 1562, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279137478210032} +{"call": 1563, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02712567476555705} +{"call": 1564, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029652560595422983} +{"call": 1565, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05176560115069151} +{"call": 1566, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030188861303031445} +{"call": 1567, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02936897473409772} +{"call": 1568, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027620255015790462} +{"call": 1569, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029656175058335066} +{"call": 1570, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.4311899026695676e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04915433004498482} +{"call": 1571, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02779820468276739} +{"call": 1572, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028853599913418293} +{"call": 1573, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03034332115203142} +{"call": 1574, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0277798087336123} +{"call": 1575, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02832389809191227} +{"call": 1576, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03420481691136956} +{"call": 1577, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03173537319526076} +{"call": 1578, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02824094006791711} +{"call": 1579, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029590678866952658} +{"call": 1580, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.438665862129248e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0490090730600059} +{"call": 1581, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02995429467409849} +{"call": 1582, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029687076807022095} +{"call": 1583, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027625145856291056} +{"call": 1584, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0332123632542789} +{"call": 1585, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02869494305923581} +{"call": 1586, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.08159603318199515} +{"call": 1587, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0464604371227324} +{"call": 1588, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030727671924978495} +{"call": 1589, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02946089580655098} +{"call": 1590, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.5269294229316645e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04862857889384031} +{"call": 1591, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029659413266927004} +{"call": 1592, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02852935204282403} +{"call": 1593, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028766504023224115} +{"call": 1594, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027930137235671282} +{"call": 1595, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03203459968790412} +{"call": 1596, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027930316049605608} +{"call": 1597, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027655798010528088} +{"call": 1598, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03108773287385702} +{"call": 1599, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028976227156817913} +{"call": 1600, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.501022838688784e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04894230328500271} +{"call": 1601, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02954580495133996} +{"call": 1602, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02778196893632412} +{"call": 1603, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028006904758512974} +{"call": 1604, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028086737263947725} +{"call": 1605, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027762992307543755} +{"call": 1606, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028043832164257765} +{"call": 1607, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029103824868798256} +{"call": 1608, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02814816916361451} +{"call": 1609, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0278836521320045} +{"call": 1610, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.6032282292149485e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049180506728589535} +{"call": 1611, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028041289187967777} +{"call": 1612, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03304873872548342} +{"call": 1613, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027802053838968277} +{"call": 1614, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027641400694847107} +{"call": 1615, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029306462034583092} +{"call": 1616, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031271840911358595} +{"call": 1617, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02825245587155223} +{"call": 1618, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02891671471297741} +{"call": 1619, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02910724887624383} +{"call": 1620, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.626577165430544e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048202994745224714} +{"call": 1621, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0307995961047709} +{"call": 1622, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02893177466467023} +{"call": 1623, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028477726969867945} +{"call": 1624, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030716530978679657} +{"call": 1625, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029833958949893713} +{"call": 1626, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028066058177500963} +{"call": 1627, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03053940599784255} +{"call": 1628, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02801911998540163} +{"call": 1629, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027982359286397696} +{"call": 1630, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.7070292137953248e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.1470223469659686} +{"call": 1631, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029632156249135733} +{"call": 1632, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05251107318326831} +{"call": 1633, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03268706286326051} +{"call": 1634, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028423632960766554} +{"call": 1635, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02804243005812168} +{"call": 1636, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03756145294755697} +{"call": 1637, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02791755786165595} +{"call": 1638, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03033453691750765} +{"call": 1639, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029013839084655046} +{"call": 1640, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.725244890664664e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04766213381662965} +{"call": 1641, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.3211470893584192} +{"call": 1642, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.037205489818006754} +{"call": 1643, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028126865159720182} +{"call": 1644, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770309615880251} +{"call": 1645, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032033930998295546} +{"call": 1646, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029379440005868673} +{"call": 1647, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02955187065526843} +{"call": 1648, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030386729165911674} +{"call": 1649, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0299610230140388} +{"call": 1650, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.734360312622635e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048413627315312624} +{"call": 1651, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027829554863274097} +{"call": 1652, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032052611000835896} +{"call": 1653, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027864702977240086} +{"call": 1654, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02818059315904975} +{"call": 1655, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029845544137060642} +{"call": 1656, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029178292956203222} +{"call": 1657, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027818862814456224} +{"call": 1658, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029200735967606306} +{"call": 1659, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030901012010872364} +{"call": 1660, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.878298033755027e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04926253389567137} +{"call": 1661, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028444823808968067} +{"call": 1662, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027806207071989775} +{"call": 1663, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0297326548025012} +{"call": 1664, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030290077906101942} +{"call": 1665, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0283960928209126} +{"call": 1666, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03125917399302125} +{"call": 1667, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02831973973661661} +{"call": 1668, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03241095785051584} +{"call": 1669, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032379355281591415} +{"call": 1670, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.881537976885637e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047548540867865086} +{"call": 1671, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030572111252695322} +{"call": 1672, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032458463218063116} +{"call": 1673, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029465830884873867} +{"call": 1674, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029337970074266195} +{"call": 1675, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02761000022292137} +{"call": 1676, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0300781880505383} +{"call": 1677, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030693504959344864} +{"call": 1678, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027771648950874805} +{"call": 1679, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03028518194332719} +{"call": 1680, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.8339037737265364e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05363530805334449} +{"call": 1681, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028624102007597685} +{"call": 1682, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027845599222928286} +{"call": 1683, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027896482031792402} +{"call": 1684, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279303090646863} +{"call": 1685, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02892669104039669} +{"call": 1686, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027894502971321344} +{"call": 1687, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02863101987168193} +{"call": 1688, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02780721802264452} +{"call": 1689, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02960237395018339} +{"call": 1690, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.8033128376591563e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048211284913122654} +{"call": 1691, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02856815606355667} +{"call": 1692, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03263981407508254} +{"call": 1693, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028226171154528856} +{"call": 1694, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028793332166969776} +{"call": 1695, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02862625801935792} +{"call": 1696, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031538814306259155} +{"call": 1697, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02782974299043417} +{"call": 1698, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028005009051412344} +{"call": 1699, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027913942001760006} +{"call": 1700, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.8368278927772165e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0531183248385787} +{"call": 1701, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031039576046168804} +{"call": 1702, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02890005288645625} +{"call": 1703, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028638835065066814} +{"call": 1704, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03314973181113601} +{"call": 1705, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029220283962786198} +{"call": 1706, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02820248482748866} +{"call": 1707, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031026022043079138} +{"call": 1708, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027870241086930037} +{"call": 1709, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05307736387476325} +{"call": 1710, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.874902394205841e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04792782012373209} +{"call": 1711, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03107335977256298} +{"call": 1712, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027831850107759237} +{"call": 1713, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027973446995019913} +{"call": 1714, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03096881415694952} +{"call": 1715, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030132493004202843} +{"call": 1716, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028207785915583372} +{"call": 1717, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027902582194656134} +{"call": 1718, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030500785913318396} +{"call": 1719, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02881439682096243} +{"call": 1720, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.8028980826528795e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04985529603436589} +{"call": 1721, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02927818102762103} +{"call": 1722, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027623181231319904} +{"call": 1723, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02823919104412198} +{"call": 1724, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030469847843050957} +{"call": 1725, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027834742795675993} +{"call": 1726, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792302379384637} +{"call": 1727, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031637981068342924} +{"call": 1728, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02800538996234536} +{"call": 1729, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028117846231907606} +{"call": 1730, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.758282042380983e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048553005792200565} +{"call": 1731, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028662275057286024} +{"call": 1732, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028691827785223722} +{"call": 1733, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03175968397408724} +{"call": 1734, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789486339315772} +{"call": 1735, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027969321701675653} +{"call": 1736, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029695323202759027} +{"call": 1737, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0278593422845006} +{"call": 1738, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030707252211868763} +{"call": 1739, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028789013158529997} +{"call": 1740, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.774925876735008e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05452314484864473} +{"call": 1741, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03075010608881712} +{"call": 1742, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03454466490074992} +{"call": 1743, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028316176030784845} +{"call": 1744, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027739700861275196} +{"call": 1745, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03344121063128114} +{"call": 1746, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279719908721745} +{"call": 1747, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02995505090802908} +{"call": 1748, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02967367321252823} +{"call": 1749, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02943472471088171} +{"call": 1750, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.819826911244295e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04975930508226156} +{"call": 1751, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02851433726027608} +{"call": 1752, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05354601098224521} +{"call": 1753, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027891858015209436} +{"call": 1754, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033135341946035624} +{"call": 1755, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03159953001886606} +{"call": 1756, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028050461784005165} +{"call": 1757, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030182767193764448} +{"call": 1758, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028569686226546764} +{"call": 1759, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03808106016367674} +{"call": 1760, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.973668720888853e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048617079854011536} +{"call": 1761, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028133951127529144} +{"call": 1762, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029079590924084187} +{"call": 1763, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0280046290718019} +{"call": 1764, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027994167990982533} +{"call": 1765, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030339448247104883} +{"call": 1766, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772808726876974} +{"call": 1767, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03229210898280144} +{"call": 1768, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027880201116204262} +{"call": 1769, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027798345778137445} +{"call": 1770, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.912495891726726e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04928109375759959} +{"call": 1771, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027935785707086325} +{"call": 1772, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02946386206895113} +{"call": 1773, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0283449599519372} +{"call": 1774, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028906756080687046} +{"call": 1775, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02776263188570738} +{"call": 1776, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029393463861197233} +{"call": 1777, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03287274483591318} +{"call": 1778, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029114627745002508} +{"call": 1779, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027713200077414513} +{"call": 1780, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.994668657828246e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0493118679150939} +{"call": 1781, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02897949330508709} +{"call": 1782, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027801050804555416} +{"call": 1783, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028456290252506733} +{"call": 1784, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027969892602413893} +{"call": 1785, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029699023813009262} +{"call": 1786, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027831423096358776} +{"call": 1787, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02777504315599799} +{"call": 1788, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029412993229925632} +{"call": 1789, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02923474693670869} +{"call": 1790, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.927516917875529e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049421084113419056} +{"call": 1791, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029159605968743563} +{"call": 1792, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05788594391196966} +{"call": 1793, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02825383096933365} +{"call": 1794, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0370518802665174} +{"call": 1795, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029474745970219374} +{"call": 1796, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02785916207358241} +{"call": 1797, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03026136104017496} +{"call": 1798, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027651197742670774} +{"call": 1799, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0294410502538085} +{"call": 1800, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 3.89420536916895e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04922124883159995} +{"call": 1801, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02798403287306428} +{"call": 1802, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03412164468318224} +{"call": 1803, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027598469983786345} +{"call": 1804, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028015898074954748} +{"call": 1805, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02772780181840062} +{"call": 1806, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029557164758443832} +{"call": 1807, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027662565000355244} +{"call": 1808, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02771735703572631} +{"call": 1809, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028091933112591505} +{"call": 1810, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.044325902696957e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04951170505955815} +{"call": 1811, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02876217197626829} +{"call": 1812, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027842445764690638} +{"call": 1813, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02894895989447832} +{"call": 1814, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027784216217696667} +{"call": 1815, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02800073381513357} +{"call": 1816, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030938480980694294} +{"call": 1817, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02835438633337617} +{"call": 1818, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027778995223343372} +{"call": 1819, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027934997342526913} +{"call": 1820, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.101532852917912e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04918002104386687} +{"call": 1821, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02841498702764511} +{"call": 1822, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027983013074845076} +{"call": 1823, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02764092106372118} +{"call": 1824, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03415948199108243} +{"call": 1825, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770040789619088} +{"call": 1826, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03002566797658801} +{"call": 1827, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03244591783732176} +{"call": 1828, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.053671285044401884} +{"call": 1829, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028047197964042425} +{"call": 1830, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.089106657799185e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048842953983694315} +{"call": 1831, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027949185110628605} +{"call": 1832, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027706341817975044} +{"call": 1833, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02791079320013523} +{"call": 1834, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027978722006082535} +{"call": 1835, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030746072996407747} +{"call": 1836, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027688536327332258} +{"call": 1837, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027800373267382383} +{"call": 1838, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027691128198057413} +{"call": 1839, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029652641154825687} +{"call": 1840, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.15027561990037e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04787132702767849} +{"call": 1841, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028955040965229273} +{"call": 1842, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775075798854232} +{"call": 1843, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027710420079529285} +{"call": 1844, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04285096004605293} +{"call": 1845, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028704876080155373} +{"call": 1846, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02781990636140108} +{"call": 1847, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028139909729361534} +{"call": 1848, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027701948769390583} +{"call": 1849, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028827050235122442} +{"call": 1850, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.182864935526908e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04880780912935734} +{"call": 1851, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028865769039839506} +{"call": 1852, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031306070275604725} +{"call": 1853, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02947383327409625} +{"call": 1854, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030874661169946194} +{"call": 1855, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029153627809137106} +{"call": 1856, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757971128448844} +{"call": 1857, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.055896558333188295} +{"call": 1858, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028483674861490726} +{"call": 1859, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279244021512568} +{"call": 1860, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.225356390743475e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04785065585747361} +{"call": 1861, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027654898818582296} +{"call": 1862, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027815162669867277} +{"call": 1863, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029927129857242107} +{"call": 1864, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028915940783917904} +{"call": 1865, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03764851810410619} +{"call": 1866, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789334813132882} +{"call": 1867, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028452987782657146} +{"call": 1868, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03069584211334586} +{"call": 1869, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027810944709926844} +{"call": 1870, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.237545592480023e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0485049718990922} +{"call": 1871, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030235149897634983} +{"call": 1872, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787583600729704} +{"call": 1873, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03044249676167965} +{"call": 1874, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031110999174416065} +{"call": 1875, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792663313448429} +{"call": 1876, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028080568183213472} +{"call": 1877, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029275465290993452} +{"call": 1878, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031173354014754295} +{"call": 1879, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027767118997871876} +{"call": 1880, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.408552575524853e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04976850003004074} +{"call": 1881, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03161637997254729} +{"call": 1882, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799248229712248} +{"call": 1883, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027876222971826792} +{"call": 1884, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03423187881708145} +{"call": 1885, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028765362221747637} +{"call": 1886, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030316094867885113} +{"call": 1887, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027872763108462095} +{"call": 1888, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027896405197679996} +{"call": 1889, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02774437563493848} +{"call": 1890, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.3552817162358e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04885442182421684} +{"call": 1891, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02883612224832177} +{"call": 1892, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027848599944263697} +{"call": 1893, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027838301844894886} +{"call": 1894, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030364217702299356} +{"call": 1895, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027753248810768127} +{"call": 1896, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02830842137336731} +{"call": 1897, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031192199792712927} +{"call": 1898, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03342222422361374} +{"call": 1899, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027741489931941032} +{"call": 1900, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.488039764490237e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049059226643294096} +{"call": 1901, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03920850809663534} +{"call": 1902, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03172021731734276} +{"call": 1903, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02817580197006464} +{"call": 1904, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028430546168237925} +{"call": 1905, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03237542789429426} +{"call": 1906, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027600686065852642} +{"call": 1907, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02794797532260418} +{"call": 1908, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03405176242813468} +{"call": 1909, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03381321998313069} +{"call": 1910, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.515923452724707e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049119307193905115} +{"call": 1911, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033568630926311016} +{"call": 1912, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032917415257543325} +{"call": 1913, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02879983512684703} +{"call": 1914, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028812550008296967} +{"call": 1915, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02821765374392271} +{"call": 1916, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027926514856517315} +{"call": 1917, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031458362005651} +{"call": 1918, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027862548828125} +{"call": 1919, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030104466248303652} +{"call": 1920, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.423266742049084e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04722598474472761} +{"call": 1921, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030835350044071674} +{"call": 1922, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03039083629846573} +{"call": 1923, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028600571677088737} +{"call": 1924, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0281846490688622} +{"call": 1925, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02760417526587844} +{"call": 1926, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029420381877571344} +{"call": 1927, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02967194002121687} +{"call": 1928, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028781349770724773} +{"call": 1929, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02780080819502473} +{"call": 1930, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.557149811962315e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04865212878212333} +{"call": 1931, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028441411908715963} +{"call": 1932, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027863603085279465} +{"call": 1933, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02925157779827714} +{"call": 1934, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030556316021829844} +{"call": 1935, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027881670743227005} +{"call": 1936, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029739588033407927} +{"call": 1937, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028958203736692667} +{"call": 1938, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028218267019838095} +{"call": 1939, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028271370101720095} +{"call": 1940, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.44419067065714e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05170581582933664} +{"call": 1941, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03061104193329811} +{"call": 1942, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028921775985509157} +{"call": 1943, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02777593396604061} +{"call": 1944, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033367861062288284} +{"call": 1945, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0277530150488019} +{"call": 1946, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03144706878811121} +{"call": 1947, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028373404871672392} +{"call": 1948, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027916083112359047} +{"call": 1949, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027725575026124716} +{"call": 1950, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.427844531755241e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04865226522088051} +{"call": 1951, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028720061760395765} +{"call": 1952, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02757558785378933} +{"call": 1953, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030923022888600826} +{"call": 1954, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02928798273205757} +{"call": 1955, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029481018893420696} +{"call": 1956, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030779660679399967} +{"call": 1957, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02769628632813692} +{"call": 1958, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029024288058280945} +{"call": 1959, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02815429726615548} +{"call": 1960, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.529696853093399e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04823228111490607} +{"call": 1961, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028123104013502598} +{"call": 1962, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027782580815255642} +{"call": 1963, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029419538099318743} +{"call": 1964, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027884410228580236} +{"call": 1965, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787606231868267} +{"call": 1966, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02806704817339778} +{"call": 1967, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032760570757091045} +{"call": 1968, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027960414066910744} +{"call": 1969, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787579922005534} +{"call": 1970, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.515054945672373e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04932041605934501} +{"call": 1971, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03269378561526537} +{"call": 1972, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031943118665367365} +{"call": 1973, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028661062940955162} +{"call": 1974, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028943865559995174} +{"call": 1975, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02902473835274577} +{"call": 1976, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028301180340349674} +{"call": 1977, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030298553872853518} +{"call": 1978, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030979336239397526} +{"call": 1979, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02792523382231593} +{"call": 1980, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.664241951576037e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0487642758525908} +{"call": 1981, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02802579803392291} +{"call": 1982, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029580652713775635} +{"call": 1983, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029581178911030293} +{"call": 1984, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0297278449870646} +{"call": 1985, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030479162000119686} +{"call": 1986, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029775322880595922} +{"call": 1987, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03132842807099223} +{"call": 1988, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029713678173720837} +{"call": 1989, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027971344999969006} +{"call": 1990, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.574335669282065e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04884025268256664} +{"call": 1991, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02980652265250683} +{"call": 1992, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.048600835260003805} +{"call": 1993, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027901488821953535} +{"call": 1994, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03199057886376977} +{"call": 1995, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028451791033148766} +{"call": 1996, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030017378740012646} +{"call": 1997, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03171275416389108} +{"call": 1998, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03352288203313947} +{"call": 1999, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028783520683646202} +{"call": 2000, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.581238668104227e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048656302969902754} +{"call": 2001, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030347938183695078} +{"call": 2002, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028068047016859055} +{"call": 2003, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02810213714838028} +{"call": 2004, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027795945294201374} +{"call": 2005, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279944590292871} +{"call": 2006, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02788857975974679} +{"call": 2007, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02810042817145586} +{"call": 2008, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029520876239985228} +{"call": 2009, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029245541896671057} +{"call": 2010, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.610627227969049e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.047661783173680305} +{"call": 2011, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02923989901319146} +{"call": 2012, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028031403198838234} +{"call": 2013, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029174827970564365} +{"call": 2014, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029134151060134172} +{"call": 2015, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028234608005732298} +{"call": 2016, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028101535979658365} +{"call": 2017, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03353620180860162} +{"call": 2018, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028007521759718657} +{"call": 2019, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775697922334075} +{"call": 2020, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.618469594869965e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049252589233219624} +{"call": 2021, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03148960089311004} +{"call": 2022, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027822660747915506} +{"call": 2023, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027737000957131386} +{"call": 2024, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031769461929798126} +{"call": 2025, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028082503005862236} +{"call": 2026, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029416579753160477} +{"call": 2027, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03373940195888281} +{"call": 2028, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027973740827292204} +{"call": 2029, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028227363247424364} +{"call": 2030, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.590225398411145e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049300095066428185} +{"call": 2031, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787548629567027} +{"call": 2032, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030209174379706383} +{"call": 2033, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028003635350614786} +{"call": 2034, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029859425034374} +{"call": 2035, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029675188940018415} +{"call": 2036, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02911236695945263} +{"call": 2037, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03034663712605834} +{"call": 2038, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02993873693048954} +{"call": 2039, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030064701102674007} +{"call": 2040, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.69576473233356e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04864953691139817} +{"call": 2041, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02816990204155445} +{"call": 2042, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033217563293874264} +{"call": 2043, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028000162914395332} +{"call": 2044, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029060238040983677} +{"call": 2045, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03219003928825259} +{"call": 2046, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02781219920143485} +{"call": 2047, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03544993419200182} +{"call": 2048, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03110599424690008} +{"call": 2049, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0300990529358387} +{"call": 2050, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.687702468984789e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04741274286061525} +{"call": 2051, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027978571131825447} +{"call": 2052, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02899125311523676} +{"call": 2053, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030029944144189358} +{"call": 2054, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02828948711976409} +{"call": 2055, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027842913754284382} +{"call": 2056, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028977838810533285} +{"call": 2057, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03579421387985349} +{"call": 2058, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028178065083920956} +{"call": 2059, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027939041145145893} +{"call": 2060, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.676068252611321e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04878704994916916} +{"call": 2061, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029743424151092768} +{"call": 2062, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027811733074486256} +{"call": 2063, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027992545161396265} +{"call": 2064, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03138608392328024} +{"call": 2065, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028006731998175383} +{"call": 2066, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028548951260745525} +{"call": 2067, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029073128011077642} +{"call": 2068, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028092594351619482} +{"call": 2069, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03164401976391673} +{"call": 2070, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.696559990626523e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048908837139606476} +{"call": 2071, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028029833920300007} +{"call": 2072, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03171864803880453} +{"call": 2073, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027962855994701385} +{"call": 2074, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0310029243119061} +{"call": 2075, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03015603730455041} +{"call": 2076, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03208499727770686} +{"call": 2077, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02976841712370515} +{"call": 2078, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03197765024378896} +{"call": 2079, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028368889819830656} +{"call": 2080, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.66929514133178e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04867547191679478} +{"call": 2081, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03070919867604971} +{"call": 2082, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775674406439066} +{"call": 2083, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027901367750018835} +{"call": 2084, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02818173822015524} +{"call": 2085, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02952303411439061} +{"call": 2086, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028014782816171646} +{"call": 2087, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03165889112278819} +{"call": 2088, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028985709883272648} +{"call": 2089, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027669763192534447} +{"call": 2090, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.729961469564011e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04865010501816869} +{"call": 2091, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03050854429602623} +{"call": 2092, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031038513872772455} +{"call": 2093, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02770964801311493} +{"call": 2094, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027778444346040487} +{"call": 2095, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030513035133481026} +{"call": 2096, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02778891706839204} +{"call": 2097, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028555758763104677} +{"call": 2098, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027703280095010996} +{"call": 2099, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02793715987354517} +{"call": 2100, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.801743798180412e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04931940184906125} +{"call": 2101, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05248346412554383} +{"call": 2102, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033973034005612135} +{"call": 2103, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03059451002627611} +{"call": 2104, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02787900436669588} +{"call": 2105, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029057654086500406} +{"call": 2106, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0323246237821877} +{"call": 2107, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029320145025849342} +{"call": 2108, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028402627911418676} +{"call": 2109, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03344388399273157} +{"call": 2110, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.723051535841092e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04933286504819989} +{"call": 2111, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02797000389546156} +{"call": 2112, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029814260080456734} +{"call": 2113, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028706193901598454} +{"call": 2114, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03979700105264783} +{"call": 2115, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030717977322638035} +{"call": 2116, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02775549842044711} +{"call": 2117, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03197667934000492} +{"call": 2118, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027801893185824156} +{"call": 2119, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027724890038371086} +{"call": 2120, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.735910598123652e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04715255508199334} +{"call": 2121, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028065697755664587} +{"call": 2122, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027928398922085762} +{"call": 2123, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02952045574784279} +{"call": 2124, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028666967060416937} +{"call": 2125, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02897142292931676} +{"call": 2126, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031535611022263765} +{"call": 2127, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028438420966267586} +{"call": 2128, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03029200667515397} +{"call": 2129, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02999555692076683} +{"call": 2130, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.699472761653875e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04885382438078523} +{"call": 2131, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028158093336969614} +{"call": 2132, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03361220518127084} +{"call": 2133, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029166446067392826} +{"call": 2134, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028310814406722784} +{"call": 2135, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030158963054418564} +{"call": 2136, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027638325933367014} +{"call": 2137, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027735999319702387} +{"call": 2138, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.36225097300484776} +{"call": 2139, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029243180062621832} +{"call": 2140, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.65861981317956e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049290691036731005} +{"call": 2141, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030496696010231972} +{"call": 2142, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027962397783994675} +{"call": 2143, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031841977033764124} +{"call": 2144, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027636717073619366} +{"call": 2145, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028225015848875046} +{"call": 2146, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030875022057443857} +{"call": 2147, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02853701589629054} +{"call": 2148, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028101905714720488} +{"call": 2149, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027932920027524233} +{"call": 2150, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.663041641273778e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04953733691945672} +{"call": 2151, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05317818420007825} +{"call": 2152, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02811460569500923} +{"call": 2153, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029126442037522793} +{"call": 2154, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04088263399899006} +{"call": 2155, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029554590117186308} +{"call": 2156, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02929631294682622} +{"call": 2157, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027867735363543034} +{"call": 2158, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03229476511478424} +{"call": 2159, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027943336870521307} +{"call": 2160, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.726265172323197e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04864014172926545} +{"call": 2161, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02981852274388075} +{"call": 2162, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02816233364865184} +{"call": 2163, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029332152102142572} +{"call": 2164, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02815232938155532} +{"call": 2165, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028600153047591448} +{"call": 2166, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0333697977475822} +{"call": 2167, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028254346922039986} +{"call": 2168, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027740659657865763} +{"call": 2169, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027894258964806795} +{"call": 2170, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.710934979900374e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0485829021781683} +{"call": 2171, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029120217077434063} +{"call": 2172, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02915287297219038} +{"call": 2173, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03036122629418969} +{"call": 2174, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028046437073498964} +{"call": 2175, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799121616408229} +{"call": 2176, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03051607310771942} +{"call": 2177, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030683018267154694} +{"call": 2178, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02822408312931657} +{"call": 2179, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027945763897150755} +{"call": 2180, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.755431324180301e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04935407778248191} +{"call": 2181, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027983665000647306} +{"call": 2182, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02971948077902198} +{"call": 2183, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02916859881952405} +{"call": 2184, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028033043257892132} +{"call": 2185, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030706829391419888} +{"call": 2186, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02885284461081028} +{"call": 2187, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027789692860096693} +{"call": 2188, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029599454253911972} +{"call": 2189, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027722139842808247} +{"call": 2190, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.848667926229952e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04663527803495526} +{"call": 2191, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029009971767663956} +{"call": 2192, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027908650692552328} +{"call": 2193, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04086687695235014} +{"call": 2194, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027766869869083166} +{"call": 2195, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028773517813533545} +{"call": 2196, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029509754851460457} +{"call": 2197, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027897594030946493} +{"call": 2198, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027550480794161558} +{"call": 2199, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02892055083066225} +{"call": 2200, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.923752216071339e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0470879515632987} +{"call": 2201, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029062996618449688} +{"call": 2202, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02829423313960433} +{"call": 2203, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029628800228238106} +{"call": 2204, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027718023862689734} +{"call": 2205, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03211862267926335} +{"call": 2206, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028988617938011885} +{"call": 2207, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02798877190798521} +{"call": 2208, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028016437776386738} +{"call": 2209, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0313244522549212} +{"call": 2210, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.909076819564346e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04938132222741842} +{"call": 2211, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027797892224043608} +{"call": 2212, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028486052062362432} +{"call": 2213, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029076427221298218} +{"call": 2214, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028525344096124172} +{"call": 2215, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028607284650206566} +{"call": 2216, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02780646225437522} +{"call": 2217, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027671210002154112} +{"call": 2218, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028077146969735622} +{"call": 2219, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031522572971880436} +{"call": 2220, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.985513062014908e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.049700583796948195} +{"call": 2221, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02798769623041153} +{"call": 2222, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029858185909688473} +{"call": 2223, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02801720378920436} +{"call": 2224, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02837394131347537} +{"call": 2225, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02748013800010085} +{"call": 2226, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02803464513272047} +{"call": 2227, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02940124412998557} +{"call": 2228, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027885962277650833} +{"call": 2229, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029289581812918186} +{"call": 2230, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.934290573738649e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04879966098815203} +{"call": 2231, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02796516614034772} +{"call": 2232, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.5146338329650462} +{"call": 2233, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.037355033215135336} +{"call": 2234, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027782937977463007} +{"call": 2235, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027976862154901028} +{"call": 2236, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029828802682459354} +{"call": 2237, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027888406068086624} +{"call": 2238, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028913176152855158} +{"call": 2239, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029164420906454325} +{"call": 2240, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.982601033035817e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04874146403744817} +{"call": 2241, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029398977290838957} +{"call": 2242, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028025846928358078} +{"call": 2243, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028184845112264156} +{"call": 2244, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02834231499582529} +{"call": 2245, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028371309861540794} +{"call": 2246, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03108857898041606} +{"call": 2247, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030254304874688387} +{"call": 2248, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029938096646219492} +{"call": 2249, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0279493173584342} +{"call": 2250, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.985863486782703e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05032484699040651} +{"call": 2251, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027909678872674704} +{"call": 2252, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031823341734707355} +{"call": 2253, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027947349939495325} +{"call": 2254, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030207321979105473} +{"call": 2255, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02943949308246374} +{"call": 2256, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028427254874259233} +{"call": 2257, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030354605987668037} +{"call": 2258, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030978800728917122} +{"call": 2259, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.053635523188859224} +{"call": 2260, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.033049134567982e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04946594312787056} +{"call": 2261, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02793330466374755} +{"call": 2262, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02777391765266657} +{"call": 2263, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03271862491965294} +{"call": 2264, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027911041863262653} +{"call": 2265, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027940489817410707} +{"call": 2266, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03285131976008415} +{"call": 2267, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028047431260347366} +{"call": 2268, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027680312748998404} +{"call": 2269, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04888185393065214} +{"call": 2270, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.043030144341748e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048889470752328634} +{"call": 2271, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844997588545084} +{"call": 2272, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03244146518409252} +{"call": 2273, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032080726232379675} +{"call": 2274, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04769800277426839} +{"call": 2275, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.053785188123583794} +{"call": 2276, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02843136014416814} +{"call": 2277, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027996569871902466} +{"call": 2278, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029663683846592903} +{"call": 2279, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028141732327640057} +{"call": 2280, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.079921721181201e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04856443917378783} +{"call": 2281, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028842275962233543} +{"call": 2282, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02808219287544489} +{"call": 2283, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029639635235071182} +{"call": 2284, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028217175975441933} +{"call": 2285, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028013361617922783} +{"call": 2286, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02767836395651102} +{"call": 2287, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02886941423639655} +{"call": 2288, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03055978985503316} +{"call": 2289, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028227167204022408} +{"call": 2290, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 4.96762248738883e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04897741787135601} +{"call": 2291, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02964910212904215} +{"call": 2292, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02897995477542281} +{"call": 2293, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028097817674279213} +{"call": 2294, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02764266123995185} +{"call": 2295, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028700007125735283} +{"call": 2296, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029032473918050528} +{"call": 2297, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029077152255922556} +{"call": 2298, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031216570176184177} +{"call": 2299, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02789109107106924} +{"call": 2300, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.16335428898089e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.05062276590615511} +{"call": 2301, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028983965050429106} +{"call": 2302, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032150087878108025} +{"call": 2303, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0325707932934165} +{"call": 2304, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02786764595657587} +{"call": 2305, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03082565823569894} +{"call": 2306, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029762569814920425} +{"call": 2307, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0281849829480052} +{"call": 2308, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03911701496690512} +{"call": 2309, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02785868290811777} +{"call": 2310, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.206800105326526e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04818330705165863} +{"call": 2311, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028225222136825323} +{"call": 2312, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030047415755689144} +{"call": 2313, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03002585330978036} +{"call": 2314, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028021335136145353} +{"call": 2315, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027851947117596865} +{"call": 2316, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02781047997996211} +{"call": 2317, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032367255073040724} +{"call": 2318, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.030077229253947735} +{"call": 2319, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027762968093156815} +{"call": 2320, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.211099616034767e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.0486281830817461} +{"call": 2321, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03099515102803707} +{"call": 2322, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027752809692174196} +{"call": 2323, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03016813611611724} +{"call": 2324, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02981271967291832} +{"call": 2325, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027790579944849014} +{"call": 2326, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0278885867446661} +{"call": 2327, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03364340215921402} +{"call": 2328, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028516715858131647} +{"call": 2329, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03138057887554169} +{"call": 2330, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.170447694172203e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048382523003965616} +{"call": 2331, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027603499591350555} +{"call": 2332, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04104449599981308} +{"call": 2333, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029923844151198864} +{"call": 2334, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028005325701087713} +{"call": 2335, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029227802995592356} +{"call": 2336, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03158881375566125} +{"call": 2337, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029355037957429886} +{"call": 2338, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028680725023150444} +{"call": 2339, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02797565795481205} +{"call": 2340, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.214033444920047e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04946806002408266} +{"call": 2341, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027774708811193705} +{"call": 2342, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02847322914749384} +{"call": 2343, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02791135385632515} +{"call": 2344, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0296504907310009} +{"call": 2345, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027916077058762312} +{"call": 2346, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028827630914747715} +{"call": 2347, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028985431883484125} +{"call": 2348, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027785842772573233} +{"call": 2349, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028721285052597523} +{"call": 2350, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.0968040066483465e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.04917085915803909} +{"call": 2351, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029030697885900736} +{"call": 2352, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027668362949043512} +{"call": 2353, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027959262020885944} +{"call": 2354, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02798977494239807} +{"call": 2355, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03045383607968688} +{"call": 2356, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02799961995333433} +{"call": 2357, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02779985498636961} +{"call": 2358, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.0331883430480957} +{"call": 2359, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027844012714922428} +{"call": 2360, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.157179490995773e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.050929876044392586} +{"call": 2361, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028108034282922745} +{"call": 2362, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.050895921885967255} +{"call": 2363, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.031182205770164728} +{"call": 2364, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02779056504368782} +{"call": 2365, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029813149012625217} +{"call": 2366, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.029661242850124836} +{"call": 2367, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02857513027265668} +{"call": 2368, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03058149293065071} +{"call": 2369, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028174083214253187} +{"call": 2370, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.137135765305613e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.048425526823848486} +{"call": 2371, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03378515597432852} +{"call": 2372, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.03255271492525935} +{"call": 2373, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027910158038139343} +{"call": 2374, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.05254788836464286} +{"call": 2375, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.033733381889760494} +{"call": 2376, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.02781064109876752} +{"call": 2377, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027930390555411577} +{"call": 2378, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.04649813286960125} +{"call": 2379, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027813893742859364} +{"call": 2380, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "max_relative_radius_drift": 5.176556458902314e-06, "zero_radius_matrices": 0, "native_plus_adamh_step_seconds": 0.050517166033387184} +{"call": 2381, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.028004640713334084} +{"call": 2382, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027731042820960283} +{"call": 2383, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.027733635157346725} +{"call": 2384, "label": "optimizer0", "matrices": 471, "native_state_preserved": true, "native_plus_adamh_step_seconds": 0.032017077784985304} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/optimizer0_rank0000_radii.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/optimizer0_rank0000_radii.json new file mode 100644 index 0000000000000000000000000000000000000000..1a996cf5338e7e8e098450cba76ce82559e52a9a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/optimizer0_rank0000_radii.json @@ -0,0 +1,473 @@ +{ + "module.module.decoder.layers.0.mlp.linear_fc1.weight::matrix0": 196.01471401487447, + "module.module.decoder.layers.0.mlp.linear_fc2.weight::matrix0": 144.96284813723244, + "module.module.decoder.layers.0.self_attention.a_proj.weight::matrix0": 13.944426918875958, + "module.module.decoder.layers.0.self_attention.b_proj.weight::matrix0": 8.517846297841082, + "module.module.decoder.layers.0.self_attention.k_proj.weight::matrix0": 43.74937158561677, + "module.module.decoder.layers.0.self_attention.out_proj.weight::matrix0": 81.64288270773726, + "module.module.decoder.layers.0.self_attention.q_proj.weight::matrix0": 43.70081222089357, + "module.module.decoder.layers.0.self_attention.v_proj.weight::matrix0": 99.24692123311726, + "module.module.decoder.layers.0.self_attention.z_proj.weight::matrix0": 90.20229758598379, + "module.module.decoder.layers.1.mlp.linear_fc1.weight::matrix0": 206.61061407357752, + "module.module.decoder.layers.1.mlp.linear_fc2.weight::matrix0": 144.72194188215596, + "module.module.decoder.layers.1.self_attention.a_proj.weight::matrix0": 12.84982679393457, + "module.module.decoder.layers.1.self_attention.b_proj.weight::matrix0": 10.64114975860425, + "module.module.decoder.layers.1.self_attention.k_proj.weight::matrix0": 49.04642570801451, + "module.module.decoder.layers.1.self_attention.out_proj.weight::matrix0": 91.73515336343881, + "module.module.decoder.layers.1.self_attention.q_proj.weight::matrix0": 49.895870325058056, + "module.module.decoder.layers.1.self_attention.v_proj.weight::matrix0": 101.55702094843808, + "module.module.decoder.layers.1.self_attention.z_proj.weight::matrix0": 96.40360146695413, + "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight::matrix0": 71.75647685656587, + "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight::matrix0": 98.63072646742475, + "module.module.decoder.layers.10.self_attention.a_proj.weight::matrix0": 13.397864140685318, + "module.module.decoder.layers.10.self_attention.b_proj.weight::matrix0": 9.608057453397532, + "module.module.decoder.layers.10.self_attention.k_proj.weight::matrix0": 51.76573811855978, + "module.module.decoder.layers.10.self_attention.out_proj.weight::matrix0": 91.64998999789864, + "module.module.decoder.layers.10.self_attention.q_proj.weight::matrix0": 52.36852443692534, + "module.module.decoder.layers.10.self_attention.v_proj.weight::matrix0": 102.48547991050947, + "module.module.decoder.layers.10.self_attention.z_proj.weight::matrix0": 92.35822508054973, + "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight::matrix0": 66.09979766806549, + "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight::matrix0": 92.7286324315616, + "module.module.decoder.layers.11.self_attention.a_proj.weight::matrix0": 13.234099139886963, + "module.module.decoder.layers.11.self_attention.b_proj.weight::matrix0": 10.266467244449306, + "module.module.decoder.layers.11.self_attention.k_proj.weight::matrix0": 49.879974070320046, + "module.module.decoder.layers.11.self_attention.out_proj.weight::matrix0": 88.69256059390932, + "module.module.decoder.layers.11.self_attention.q_proj.weight::matrix0": 50.48765955960098, + "module.module.decoder.layers.11.self_attention.v_proj.weight::matrix0": 101.97441652440136, + "module.module.decoder.layers.11.self_attention.z_proj.weight::matrix0": 90.7791514624368, + "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight::matrix0": 71.45850129323367, + "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight::matrix0": 98.96445367053519, + "module.module.decoder.layers.12.self_attention.linear_proj.weight::matrix0": 72.28025551914449, + "module.module.decoder.layers.12.self_attention.linear_qkv.weight::matrix0": 133.97147108042742, + "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight::matrix0": 72.52661672402425, + "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight::matrix0": 98.58275116557033, + "module.module.decoder.layers.13.self_attention.a_proj.weight::matrix0": 17.507811646284242, + "module.module.decoder.layers.13.self_attention.b_proj.weight::matrix0": 10.268432053228763, + "module.module.decoder.layers.13.self_attention.k_proj.weight::matrix0": 51.92601929481206, + "module.module.decoder.layers.13.self_attention.out_proj.weight::matrix0": 91.00421422055952, + "module.module.decoder.layers.13.self_attention.q_proj.weight::matrix0": 54.52162227002865, + "module.module.decoder.layers.13.self_attention.v_proj.weight::matrix0": 101.90134439734047, + "module.module.decoder.layers.13.self_attention.z_proj.weight::matrix0": 91.4289615177182, + "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight::matrix0": 71.16463984250728, + "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight::matrix0": 100.05262458781274, + "module.module.decoder.layers.14.self_attention.a_proj.weight::matrix0": 15.712850654542015, + "module.module.decoder.layers.14.self_attention.b_proj.weight::matrix0": 11.992995898614812, + "module.module.decoder.layers.14.self_attention.k_proj.weight::matrix0": 51.98172050347499, + "module.module.decoder.layers.14.self_attention.out_proj.weight::matrix0": 91.19949413516684, + "module.module.decoder.layers.14.self_attention.q_proj.weight::matrix0": 52.981399980873796, + "module.module.decoder.layers.14.self_attention.v_proj.weight::matrix0": 102.69728418003848, + "module.module.decoder.layers.14.self_attention.z_proj.weight::matrix0": 93.49050515883121, + "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight::matrix0": 70.7407693928677, + "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight::matrix0": 98.13105482969733, + "module.module.decoder.layers.15.self_attention.a_proj.weight::matrix0": 15.451685623476216, + "module.module.decoder.layers.15.self_attention.b_proj.weight::matrix0": 11.087241170622171, + "module.module.decoder.layers.15.self_attention.k_proj.weight::matrix0": 52.40099962705805, + "module.module.decoder.layers.15.self_attention.out_proj.weight::matrix0": 89.75771711870328, + "module.module.decoder.layers.15.self_attention.q_proj.weight::matrix0": 53.29496812356144, + "module.module.decoder.layers.15.self_attention.v_proj.weight::matrix0": 101.87128714280573, + "module.module.decoder.layers.15.self_attention.z_proj.weight::matrix0": 91.59624464486016, + "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight::matrix0": 69.94462454640008, + "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight::matrix0": 98.8038086139952, + "module.module.decoder.layers.16.self_attention.a_proj.weight::matrix0": 16.245355881595568, + "module.module.decoder.layers.16.self_attention.b_proj.weight::matrix0": 10.337269135054987, + "module.module.decoder.layers.16.self_attention.k_proj.weight::matrix0": 53.2849810426188, + "module.module.decoder.layers.16.self_attention.out_proj.weight::matrix0": 88.31771205526108, + "module.module.decoder.layers.16.self_attention.q_proj.weight::matrix0": 54.81536328933465, + "module.module.decoder.layers.16.self_attention.v_proj.weight::matrix0": 101.55100721080282, + "module.module.decoder.layers.16.self_attention.z_proj.weight::matrix0": 90.29783390391218, + "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight::matrix0": 69.98687799118704, + "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight::matrix0": 99.18718379853635, + "module.module.decoder.layers.17.self_attention.a_proj.weight::matrix0": 14.722884572509718, + "module.module.decoder.layers.17.self_attention.b_proj.weight::matrix0": 11.354200710969316, + "module.module.decoder.layers.17.self_attention.k_proj.weight::matrix0": 53.7502624279073, + "module.module.decoder.layers.17.self_attention.out_proj.weight::matrix0": 88.32433303107665, + "module.module.decoder.layers.17.self_attention.q_proj.weight::matrix0": 53.78141608597541, + "module.module.decoder.layers.17.self_attention.v_proj.weight::matrix0": 100.23428558351802, + "module.module.decoder.layers.17.self_attention.z_proj.weight::matrix0": 89.64373392918138, + "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight::matrix0": 68.42454376208146, + "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight::matrix0": 96.55839772920805, + "module.module.decoder.layers.18.self_attention.a_proj.weight::matrix0": 14.887414774424263, + "module.module.decoder.layers.18.self_attention.b_proj.weight::matrix0": 10.227058385534267, + "module.module.decoder.layers.18.self_attention.k_proj.weight::matrix0": 52.7784487964401, + "module.module.decoder.layers.18.self_attention.out_proj.weight::matrix0": 86.19836261887056, + "module.module.decoder.layers.18.self_attention.q_proj.weight::matrix0": 52.952017703318305, + "module.module.decoder.layers.18.self_attention.v_proj.weight::matrix0": 100.32847377211552, + "module.module.decoder.layers.18.self_attention.z_proj.weight::matrix0": 88.92015577364201, + "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight::matrix0": 63.95318258585462, + "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight::matrix0": 95.58935947415007, + "module.module.decoder.layers.19.self_attention.a_proj.weight::matrix0": 14.17932393494252, + "module.module.decoder.layers.19.self_attention.b_proj.weight::matrix0": 10.869832759424382, + "module.module.decoder.layers.19.self_attention.k_proj.weight::matrix0": 54.904841118981366, + "module.module.decoder.layers.19.self_attention.out_proj.weight::matrix0": 85.39720203929107, + "module.module.decoder.layers.19.self_attention.q_proj.weight::matrix0": 56.39253233997743, + "module.module.decoder.layers.19.self_attention.v_proj.weight::matrix0": 98.61723851833806, + "module.module.decoder.layers.19.self_attention.z_proj.weight::matrix0": 86.78457288710855, + "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight::matrix0": 75.88948839044549, + "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight::matrix0": 102.50060260566708, + "module.module.decoder.layers.2.self_attention.a_proj.weight::matrix0": 11.723214527446864, + "module.module.decoder.layers.2.self_attention.b_proj.weight::matrix0": 10.421595756667159, + "module.module.decoder.layers.2.self_attention.k_proj.weight::matrix0": 48.35034515345289, + "module.module.decoder.layers.2.self_attention.out_proj.weight::matrix0": 94.15670818683482, + "module.module.decoder.layers.2.self_attention.q_proj.weight::matrix0": 49.44559830687664, + "module.module.decoder.layers.2.self_attention.v_proj.weight::matrix0": 102.4536793725893, + "module.module.decoder.layers.2.self_attention.z_proj.weight::matrix0": 96.42288064287851, + "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight::matrix0": 66.07291984319382, + "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight::matrix0": 96.08445745911465, + "module.module.decoder.layers.20.self_attention.linear_proj.weight::matrix0": 73.77977565102759, + "module.module.decoder.layers.20.self_attention.linear_qkv.weight::matrix0": 124.14015428232752, + "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight::matrix0": 67.83767025451067, + "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight::matrix0": 100.80155569921013, + "module.module.decoder.layers.21.self_attention.linear_proj.weight::matrix0": 79.94541853028785, + "module.module.decoder.layers.21.self_attention.linear_qkv.weight::matrix0": 122.95455535517578, + "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight::matrix0": 66.09162373477464, + "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight::matrix0": 92.93545282504451, + "module.module.decoder.layers.22.self_attention.a_proj.weight::matrix0": 13.908590866395256, + "module.module.decoder.layers.22.self_attention.b_proj.weight::matrix0": 9.377683928569986, + "module.module.decoder.layers.22.self_attention.k_proj.weight::matrix0": 51.236345926623564, + "module.module.decoder.layers.22.self_attention.out_proj.weight::matrix0": 86.6985439855759, + "module.module.decoder.layers.22.self_attention.q_proj.weight::matrix0": 54.450149449862145, + "module.module.decoder.layers.22.self_attention.v_proj.weight::matrix0": 96.85052551055713, + "module.module.decoder.layers.22.self_attention.z_proj.weight::matrix0": 87.04423517374329, + "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight::matrix0": 69.48199569386716, + "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight::matrix0": 89.10349631553676, + "module.module.decoder.layers.23.self_attention.linear_proj.weight::matrix0": 88.19704300117944, + "module.module.decoder.layers.23.self_attention.linear_qkv.weight::matrix0": 119.61304104516198, + "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight::matrix0": 68.85104619921431, + "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight::matrix0": 95.10550188040884, + "module.module.decoder.layers.24.self_attention.a_proj.weight::matrix0": 14.01917358422263, + "module.module.decoder.layers.24.self_attention.b_proj.weight::matrix0": 9.80246656283613, + "module.module.decoder.layers.24.self_attention.k_proj.weight::matrix0": 52.1723854973428, + "module.module.decoder.layers.24.self_attention.out_proj.weight::matrix0": 91.89605611235993, + "module.module.decoder.layers.24.self_attention.q_proj.weight::matrix0": 55.95818283692856, + "module.module.decoder.layers.24.self_attention.v_proj.weight::matrix0": 100.6710748543814, + "module.module.decoder.layers.24.self_attention.z_proj.weight::matrix0": 88.50991481298223, + "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight::matrix0": 68.16702483573815, + "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight::matrix0": 97.55838221489581, + "module.module.decoder.layers.25.self_attention.a_proj.weight::matrix0": 13.139968553637864, + "module.module.decoder.layers.25.self_attention.b_proj.weight::matrix0": 10.098873250325243, + "module.module.decoder.layers.25.self_attention.k_proj.weight::matrix0": 52.53767309655471, + "module.module.decoder.layers.25.self_attention.out_proj.weight::matrix0": 91.96067526543457, + "module.module.decoder.layers.25.self_attention.q_proj.weight::matrix0": 54.08718702502012, + "module.module.decoder.layers.25.self_attention.v_proj.weight::matrix0": 102.1914486508032, + "module.module.decoder.layers.25.self_attention.z_proj.weight::matrix0": 90.56757399301704, + "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight::matrix0": 67.34926148083038, + "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight::matrix0": 101.86246167495443, + "module.module.decoder.layers.26.self_attention.a_proj.weight::matrix0": 15.576180138451402, + "module.module.decoder.layers.26.self_attention.b_proj.weight::matrix0": 9.93425545433866, + "module.module.decoder.layers.26.self_attention.k_proj.weight::matrix0": 52.203608858610764, + "module.module.decoder.layers.26.self_attention.out_proj.weight::matrix0": 91.76822266925137, + "module.module.decoder.layers.26.self_attention.q_proj.weight::matrix0": 54.31178224028386, + "module.module.decoder.layers.26.self_attention.v_proj.weight::matrix0": 100.41854453829937, + "module.module.decoder.layers.26.self_attention.z_proj.weight::matrix0": 90.04388636629001, + "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight::matrix0": 67.29256588825436, + "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight::matrix0": 101.49081301559411, + "module.module.decoder.layers.27.self_attention.a_proj.weight::matrix0": 12.468722672754073, + "module.module.decoder.layers.27.self_attention.b_proj.weight::matrix0": 8.360085641719756, + "module.module.decoder.layers.27.self_attention.k_proj.weight::matrix0": 47.35237284101135, + "module.module.decoder.layers.27.self_attention.out_proj.weight::matrix0": 88.79197911707004, + "module.module.decoder.layers.27.self_attention.q_proj.weight::matrix0": 49.92819876432449, + "module.module.decoder.layers.27.self_attention.v_proj.weight::matrix0": 98.033624970259, + "module.module.decoder.layers.27.self_attention.z_proj.weight::matrix0": 91.38231328684822, + "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight::matrix0": 70.72088876237486, + "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight::matrix0": 89.25266189446992, + "module.module.decoder.layers.28.self_attention.a_proj.weight::matrix0": 11.775047246190256, + "module.module.decoder.layers.28.self_attention.b_proj.weight::matrix0": 7.62458696359391, + "module.module.decoder.layers.28.self_attention.k_proj.weight::matrix0": 48.77080481927439, + "module.module.decoder.layers.28.self_attention.out_proj.weight::matrix0": 89.89377476846528, + "module.module.decoder.layers.28.self_attention.q_proj.weight::matrix0": 54.11868758186382, + "module.module.decoder.layers.28.self_attention.v_proj.weight::matrix0": 101.43940820137136, + "module.module.decoder.layers.28.self_attention.z_proj.weight::matrix0": 90.89646973548074, + "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight::matrix0": 69.03642047025042, + "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight::matrix0": 90.0421812495324, + "module.module.decoder.layers.29.self_attention.a_proj.weight::matrix0": 11.762571810832162, + "module.module.decoder.layers.29.self_attention.b_proj.weight::matrix0": 10.128977262028425, + "module.module.decoder.layers.29.self_attention.k_proj.weight::matrix0": 51.96122018886025, + "module.module.decoder.layers.29.self_attention.out_proj.weight::matrix0": 92.54525136744324, + "module.module.decoder.layers.29.self_attention.q_proj.weight::matrix0": 53.7334338561436, + "module.module.decoder.layers.29.self_attention.v_proj.weight::matrix0": 102.39001776194847, + "module.module.decoder.layers.29.self_attention.z_proj.weight::matrix0": 92.71487267361272, + "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight::matrix0": 76.6740171109891, + "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight::matrix0": 100.50529362958248, + "module.module.decoder.layers.3.self_attention.a_proj.weight::matrix0": 13.283049577956474, + "module.module.decoder.layers.3.self_attention.b_proj.weight::matrix0": 10.808082268355525, + "module.module.decoder.layers.3.self_attention.k_proj.weight::matrix0": 51.33411125331745, + "module.module.decoder.layers.3.self_attention.out_proj.weight::matrix0": 93.37660547775698, + "module.module.decoder.layers.3.self_attention.q_proj.weight::matrix0": 51.357855127834156, + "module.module.decoder.layers.3.self_attention.v_proj.weight::matrix0": 101.50715143234027, + "module.module.decoder.layers.3.self_attention.z_proj.weight::matrix0": 96.39562895391566, + "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight::matrix0": 68.32569147757489, + "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight::matrix0": 97.4024765353194, + "module.module.decoder.layers.30.self_attention.a_proj.weight::matrix0": 11.163542249537588, + "module.module.decoder.layers.30.self_attention.b_proj.weight::matrix0": 9.0532959897096, + "module.module.decoder.layers.30.self_attention.k_proj.weight::matrix0": 52.870242204473485, + "module.module.decoder.layers.30.self_attention.out_proj.weight::matrix0": 95.21091590777563, + "module.module.decoder.layers.30.self_attention.q_proj.weight::matrix0": 56.00134717564299, + "module.module.decoder.layers.30.self_attention.v_proj.weight::matrix0": 102.88645238083234, + "module.module.decoder.layers.30.self_attention.z_proj.weight::matrix0": 91.99841515453436, + "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight::matrix0": 68.02300837035231, + "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight::matrix0": 96.39043801744803, + "module.module.decoder.layers.31.self_attention.a_proj.weight::matrix0": 9.374242122218158, + "module.module.decoder.layers.31.self_attention.b_proj.weight::matrix0": 6.075155871706631, + "module.module.decoder.layers.31.self_attention.k_proj.weight::matrix0": 54.07420631715537, + "module.module.decoder.layers.31.self_attention.out_proj.weight::matrix0": 92.93692455790095, + "module.module.decoder.layers.31.self_attention.q_proj.weight::matrix0": 57.474244869295624, + "module.module.decoder.layers.31.self_attention.v_proj.weight::matrix0": 105.65321561374377, + "module.module.decoder.layers.31.self_attention.z_proj.weight::matrix0": 92.95431827084887, + "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight::matrix0": 67.99026265114462, + "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight::matrix0": 96.99133058947217, + "module.module.decoder.layers.32.self_attention.a_proj.weight::matrix0": 9.877412923906677, + "module.module.decoder.layers.32.self_attention.b_proj.weight::matrix0": 8.243481436450928, + "module.module.decoder.layers.32.self_attention.k_proj.weight::matrix0": 51.83520658791086, + "module.module.decoder.layers.32.self_attention.out_proj.weight::matrix0": 91.34112325262107, + "module.module.decoder.layers.32.self_attention.q_proj.weight::matrix0": 53.92107386374616, + "module.module.decoder.layers.32.self_attention.v_proj.weight::matrix0": 103.537237507121, + "module.module.decoder.layers.32.self_attention.z_proj.weight::matrix0": 94.8396825115097, + "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight::matrix0": 68.32969847840836, + "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight::matrix0": 95.09422353024276, + "module.module.decoder.layers.33.self_attention.a_proj.weight::matrix0": 9.266718690364069, + "module.module.decoder.layers.33.self_attention.b_proj.weight::matrix0": 7.711523734216587, + "module.module.decoder.layers.33.self_attention.k_proj.weight::matrix0": 50.29403429220021, + "module.module.decoder.layers.33.self_attention.out_proj.weight::matrix0": 89.53349030827071, + "module.module.decoder.layers.33.self_attention.q_proj.weight::matrix0": 52.542642582327, + "module.module.decoder.layers.33.self_attention.v_proj.weight::matrix0": 101.45722434887622, + "module.module.decoder.layers.33.self_attention.z_proj.weight::matrix0": 95.07711369911365, + "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight::matrix0": 66.79522648888278, + "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight::matrix0": 93.14658337776247, + "module.module.decoder.layers.34.self_attention.a_proj.weight::matrix0": 10.143498472440102, + "module.module.decoder.layers.34.self_attention.b_proj.weight::matrix0": 7.430204840001662, + "module.module.decoder.layers.34.self_attention.k_proj.weight::matrix0": 52.969907744806775, + "module.module.decoder.layers.34.self_attention.out_proj.weight::matrix0": 91.1783532336854, + "module.module.decoder.layers.34.self_attention.q_proj.weight::matrix0": 55.64062281595556, + "module.module.decoder.layers.34.self_attention.v_proj.weight::matrix0": 101.42066768322572, + "module.module.decoder.layers.34.self_attention.z_proj.weight::matrix0": 92.57887317032218, + "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight::matrix0": 66.56375127219194, + "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight::matrix0": 91.92839179246573, + "module.module.decoder.layers.35.self_attention.a_proj.weight::matrix0": 7.848294675935004, + "module.module.decoder.layers.35.self_attention.b_proj.weight::matrix0": 7.145634467207909, + "module.module.decoder.layers.35.self_attention.k_proj.weight::matrix0": 43.48050138475016, + "module.module.decoder.layers.35.self_attention.out_proj.weight::matrix0": 84.69909766691458, + "module.module.decoder.layers.35.self_attention.q_proj.weight::matrix0": 46.629866355153645, + "module.module.decoder.layers.35.self_attention.v_proj.weight::matrix0": 98.81089201800344, + "module.module.decoder.layers.35.self_attention.z_proj.weight::matrix0": 93.8529909080526, + "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight::matrix0": 68.04015882760837, + "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight::matrix0": 96.89520558191491, + "module.module.decoder.layers.36.self_attention.a_proj.weight::matrix0": 9.545382300629571, + "module.module.decoder.layers.36.self_attention.b_proj.weight::matrix0": 8.440219672197562, + "module.module.decoder.layers.36.self_attention.k_proj.weight::matrix0": 49.57313030461717, + "module.module.decoder.layers.36.self_attention.out_proj.weight::matrix0": 92.20390887346527, + "module.module.decoder.layers.36.self_attention.q_proj.weight::matrix0": 50.066898583070994, + "module.module.decoder.layers.36.self_attention.v_proj.weight::matrix0": 102.97851355602218, + "module.module.decoder.layers.36.self_attention.z_proj.weight::matrix0": 96.70473514543195, + "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight::matrix0": 67.8694218435567, + "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight::matrix0": 96.70817587785176, + "module.module.decoder.layers.37.self_attention.a_proj.weight::matrix0": 9.189579200082703, + "module.module.decoder.layers.37.self_attention.b_proj.weight::matrix0": 8.692220598246184, + "module.module.decoder.layers.37.self_attention.k_proj.weight::matrix0": 49.415496310271664, + "module.module.decoder.layers.37.self_attention.out_proj.weight::matrix0": 91.22763046961501, + "module.module.decoder.layers.37.self_attention.q_proj.weight::matrix0": 49.5173044083257, + "module.module.decoder.layers.37.self_attention.v_proj.weight::matrix0": 101.34725599827586, + "module.module.decoder.layers.37.self_attention.z_proj.weight::matrix0": 94.9173479163966, + "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight::matrix0": 66.25485886107381, + "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight::matrix0": 96.18858301294684, + "module.module.decoder.layers.38.self_attention.a_proj.weight::matrix0": 8.63009914878203, + "module.module.decoder.layers.38.self_attention.b_proj.weight::matrix0": 7.923619598591464, + "module.module.decoder.layers.38.self_attention.k_proj.weight::matrix0": 47.39466844732648, + "module.module.decoder.layers.38.self_attention.out_proj.weight::matrix0": 89.87699207548205, + "module.module.decoder.layers.38.self_attention.q_proj.weight::matrix0": 49.09703063792047, + "module.module.decoder.layers.38.self_attention.v_proj.weight::matrix0": 100.81286695310342, + "module.module.decoder.layers.38.self_attention.z_proj.weight::matrix0": 92.75195817870807, + "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight::matrix0": 66.5798313597731, + "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight::matrix0": 97.19503252224204, + "module.module.decoder.layers.39.self_attention.a_proj.weight::matrix0": 9.297597018275788, + "module.module.decoder.layers.39.self_attention.b_proj.weight::matrix0": 6.349273258997344, + "module.module.decoder.layers.39.self_attention.k_proj.weight::matrix0": 48.84946257661362, + "module.module.decoder.layers.39.self_attention.out_proj.weight::matrix0": 89.28830426131816, + "module.module.decoder.layers.39.self_attention.q_proj.weight::matrix0": 51.10425249348585, + "module.module.decoder.layers.39.self_attention.v_proj.weight::matrix0": 100.82212150044823, + "module.module.decoder.layers.39.self_attention.z_proj.weight::matrix0": 92.38942989578877, + "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight::matrix0": 73.11306471531167, + "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight::matrix0": 100.474560862248, + "module.module.decoder.layers.4.self_attention.a_proj.weight::matrix0": 12.801453335574008, + "module.module.decoder.layers.4.self_attention.b_proj.weight::matrix0": 10.695784209660891, + "module.module.decoder.layers.4.self_attention.k_proj.weight::matrix0": 50.83442234130194, + "module.module.decoder.layers.4.self_attention.out_proj.weight::matrix0": 94.31360955474506, + "module.module.decoder.layers.4.self_attention.q_proj.weight::matrix0": 51.28447048264714, + "module.module.decoder.layers.4.self_attention.v_proj.weight::matrix0": 101.47621130116936, + "module.module.decoder.layers.4.self_attention.z_proj.weight::matrix0": 96.00009364370003, + "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight::matrix0": 65.37086508242108, + "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight::matrix0": 95.82281565621017, + "module.module.decoder.layers.40.self_attention.a_proj.weight::matrix0": 8.086318527816694, + "module.module.decoder.layers.40.self_attention.b_proj.weight::matrix0": 6.458619820124073, + "module.module.decoder.layers.40.self_attention.k_proj.weight::matrix0": 49.75698207007095, + "module.module.decoder.layers.40.self_attention.out_proj.weight::matrix0": 88.98081173500529, + "module.module.decoder.layers.40.self_attention.q_proj.weight::matrix0": 50.46174501299422, + "module.module.decoder.layers.40.self_attention.v_proj.weight::matrix0": 100.96422296837213, + "module.module.decoder.layers.40.self_attention.z_proj.weight::matrix0": 91.70165005885616, + "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight::matrix0": 66.10665505887573, + "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight::matrix0": 94.48481592404053, + "module.module.decoder.layers.41.self_attention.a_proj.weight::matrix0": 9.045905525115664, + "module.module.decoder.layers.41.self_attention.b_proj.weight::matrix0": 7.180014682306411, + "module.module.decoder.layers.41.self_attention.k_proj.weight::matrix0": 49.20082143874834, + "module.module.decoder.layers.41.self_attention.out_proj.weight::matrix0": 91.27558513796754, + "module.module.decoder.layers.41.self_attention.q_proj.weight::matrix0": 49.59944610391502, + "module.module.decoder.layers.41.self_attention.v_proj.weight::matrix0": 99.95659302609361, + "module.module.decoder.layers.41.self_attention.z_proj.weight::matrix0": 92.69166874847416, + "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight::matrix0": 65.90508955493726, + "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight::matrix0": 89.90259878047554, + "module.module.decoder.layers.42.self_attention.a_proj.weight::matrix0": 7.886274380984785, + "module.module.decoder.layers.42.self_attention.b_proj.weight::matrix0": 6.699232822799429, + "module.module.decoder.layers.42.self_attention.k_proj.weight::matrix0": 49.95020365536833, + "module.module.decoder.layers.42.self_attention.out_proj.weight::matrix0": 89.30214683733954, + "module.module.decoder.layers.42.self_attention.q_proj.weight::matrix0": 50.91279830449977, + "module.module.decoder.layers.42.self_attention.v_proj.weight::matrix0": 102.63921012130444, + "module.module.decoder.layers.42.self_attention.z_proj.weight::matrix0": 91.3236133295204, + "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight::matrix0": 67.90599438427334, + "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight::matrix0": 86.13636117451097, + "module.module.decoder.layers.43.self_attention.a_proj.weight::matrix0": 8.134445759824457, + "module.module.decoder.layers.43.self_attention.b_proj.weight::matrix0": 8.227348888655605, + "module.module.decoder.layers.43.self_attention.k_proj.weight::matrix0": 50.4879390816284, + "module.module.decoder.layers.43.self_attention.out_proj.weight::matrix0": 87.26858762366857, + "module.module.decoder.layers.43.self_attention.q_proj.weight::matrix0": 52.09535300525095, + "module.module.decoder.layers.43.self_attention.v_proj.weight::matrix0": 101.49607130694733, + "module.module.decoder.layers.43.self_attention.z_proj.weight::matrix0": 91.59133337078889, + "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight::matrix0": 68.69109468703287, + "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight::matrix0": 82.17252016221734, + "module.module.decoder.layers.44.self_attention.a_proj.weight::matrix0": 7.969674201148776, + "module.module.decoder.layers.44.self_attention.b_proj.weight::matrix0": 6.489563686153883, + "module.module.decoder.layers.44.self_attention.k_proj.weight::matrix0": 50.33308196870538, + "module.module.decoder.layers.44.self_attention.out_proj.weight::matrix0": 88.08901871280939, + "module.module.decoder.layers.44.self_attention.q_proj.weight::matrix0": 53.24714956827831, + "module.module.decoder.layers.44.self_attention.v_proj.weight::matrix0": 101.82479756607572, + "module.module.decoder.layers.44.self_attention.z_proj.weight::matrix0": 92.00657497463403, + "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight::matrix0": 64.0554285970409, + "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight::matrix0": 80.12420452906134, + "module.module.decoder.layers.45.self_attention.a_proj.weight::matrix0": 8.987841569490529, + "module.module.decoder.layers.45.self_attention.b_proj.weight::matrix0": 7.996941734461278, + "module.module.decoder.layers.45.self_attention.k_proj.weight::matrix0": 49.84979300793443, + "module.module.decoder.layers.45.self_attention.out_proj.weight::matrix0": 85.16177533837302, + "module.module.decoder.layers.45.self_attention.q_proj.weight::matrix0": 51.36324491820476, + "module.module.decoder.layers.45.self_attention.v_proj.weight::matrix0": 100.1459468527375, + "module.module.decoder.layers.45.self_attention.z_proj.weight::matrix0": 93.01602294818744, + "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight::matrix0": 66.8373020214295, + "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight::matrix0": 86.23968391740028, + "module.module.decoder.layers.46.self_attention.linear_proj.weight::matrix0": 111.63384851725303, + "module.module.decoder.layers.46.self_attention.linear_qkv.weight::matrix0": 127.01791283946416, + "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight::matrix0": 64.59206019251846, + "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight::matrix0": 91.31728440879938, + "module.module.decoder.layers.47.self_attention.a_proj.weight::matrix0": 9.583096180923743, + "module.module.decoder.layers.47.self_attention.b_proj.weight::matrix0": 8.408119952378394, + "module.module.decoder.layers.47.self_attention.k_proj.weight::matrix0": 48.890252297736154, + "module.module.decoder.layers.47.self_attention.out_proj.weight::matrix0": 85.49352778412803, + "module.module.decoder.layers.47.self_attention.q_proj.weight::matrix0": 50.26889256969785, + "module.module.decoder.layers.47.self_attention.v_proj.weight::matrix0": 96.64809688427268, + "module.module.decoder.layers.47.self_attention.z_proj.weight::matrix0": 89.62318494383453, + "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight::matrix0": 64.8356809576459, + "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight::matrix0": 92.04907939200076, + "module.module.decoder.layers.48.self_attention.linear_proj.weight::matrix0": 115.20371422717837, + "module.module.decoder.layers.48.self_attention.linear_qkv.weight::matrix0": 126.83098644877688, + "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight::matrix0": 67.34186827893664, + "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight::matrix0": 73.64686957348005, + "module.module.decoder.layers.49.self_attention.linear_proj.weight::matrix0": 117.48681572387434, + "module.module.decoder.layers.49.self_attention.linear_qkv.weight::matrix0": 129.52821702990727, + "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight::matrix0": 73.88348362831667, + "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight::matrix0": 100.41646899489156, + "module.module.decoder.layers.5.self_attention.a_proj.weight::matrix0": 11.84561592083049, + "module.module.decoder.layers.5.self_attention.b_proj.weight::matrix0": 9.178256224235753, + "module.module.decoder.layers.5.self_attention.k_proj.weight::matrix0": 49.49648734775725, + "module.module.decoder.layers.5.self_attention.out_proj.weight::matrix0": 92.55408831534297, + "module.module.decoder.layers.5.self_attention.q_proj.weight::matrix0": 50.86659492486074, + "module.module.decoder.layers.5.self_attention.v_proj.weight::matrix0": 101.84338780695376, + "module.module.decoder.layers.5.self_attention.z_proj.weight::matrix0": 95.2351919033372, + "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight::matrix0": 67.73315163916872, + "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight::matrix0": 71.7637998419873, + "module.module.decoder.layers.50.self_attention.a_proj.weight::matrix0": 8.25082092157322, + "module.module.decoder.layers.50.self_attention.b_proj.weight::matrix0": 5.775679328907176, + "module.module.decoder.layers.50.self_attention.k_proj.weight::matrix0": 49.733895172013156, + "module.module.decoder.layers.50.self_attention.out_proj.weight::matrix0": 90.4384582607423, + "module.module.decoder.layers.50.self_attention.q_proj.weight::matrix0": 51.94945336901787, + "module.module.decoder.layers.50.self_attention.v_proj.weight::matrix0": 109.72628611799905, + "module.module.decoder.layers.50.self_attention.z_proj.weight::matrix0": 91.5359680148654, + "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight::matrix0": 69.52843324926194, + "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight::matrix0": 72.06160753391364, + "module.module.decoder.layers.51.self_attention.a_proj.weight::matrix0": 8.696541551885327, + "module.module.decoder.layers.51.self_attention.b_proj.weight::matrix0": 6.227572012677827, + "module.module.decoder.layers.51.self_attention.k_proj.weight::matrix0": 50.23632739769612, + "module.module.decoder.layers.51.self_attention.out_proj.weight::matrix0": 89.19575697514142, + "module.module.decoder.layers.51.self_attention.q_proj.weight::matrix0": 52.057296855955244, + "module.module.decoder.layers.51.self_attention.v_proj.weight::matrix0": 109.24756767674305, + "module.module.decoder.layers.51.self_attention.z_proj.weight::matrix0": 91.41541712637144, + "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight::matrix0": 66.9254815324096, + "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight::matrix0": 70.88966157019816, + "module.module.decoder.layers.52.self_attention.a_proj.weight::matrix0": 9.884284720372312, + "module.module.decoder.layers.52.self_attention.b_proj.weight::matrix0": 9.258749423877461, + "module.module.decoder.layers.52.self_attention.k_proj.weight::matrix0": 48.55880351491055, + "module.module.decoder.layers.52.self_attention.out_proj.weight::matrix0": 91.97822915365384, + "module.module.decoder.layers.52.self_attention.q_proj.weight::matrix0": 51.650049794548764, + "module.module.decoder.layers.52.self_attention.v_proj.weight::matrix0": 101.25040239229425, + "module.module.decoder.layers.52.self_attention.z_proj.weight::matrix0": 91.53845755707984, + "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight::matrix0": 65.06751200358636, + "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight::matrix0": 94.65752590920974, + "module.module.decoder.layers.53.self_attention.a_proj.weight::matrix0": 10.367555391311909, + "module.module.decoder.layers.53.self_attention.b_proj.weight::matrix0": 8.740813886364563, + "module.module.decoder.layers.53.self_attention.k_proj.weight::matrix0": 47.87195077861514, + "module.module.decoder.layers.53.self_attention.out_proj.weight::matrix0": 87.3446073469411, + "module.module.decoder.layers.53.self_attention.q_proj.weight::matrix0": 51.28825687308385, + "module.module.decoder.layers.53.self_attention.v_proj.weight::matrix0": 96.99237955071912, + "module.module.decoder.layers.53.self_attention.z_proj.weight::matrix0": 89.54122626389889, + "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight::matrix0": 66.57747618034561, + "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight::matrix0": 104.99339207141324, + "module.module.decoder.layers.54.self_attention.a_proj.weight::matrix0": 11.158973077540539, + "module.module.decoder.layers.54.self_attention.b_proj.weight::matrix0": 8.720886202742431, + "module.module.decoder.layers.54.self_attention.k_proj.weight::matrix0": 42.827419704067665, + "module.module.decoder.layers.54.self_attention.out_proj.weight::matrix0": 85.14683151901941, + "module.module.decoder.layers.54.self_attention.q_proj.weight::matrix0": 49.14346023990518, + "module.module.decoder.layers.54.self_attention.v_proj.weight::matrix0": 93.22419351569637, + "module.module.decoder.layers.54.self_attention.z_proj.weight::matrix0": 88.59700739783533, + "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight::matrix0": 66.57908494014221, + "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight::matrix0": 110.23111274136694, + "module.module.decoder.layers.55.self_attention.a_proj.weight::matrix0": 11.021910688569937, + "module.module.decoder.layers.55.self_attention.b_proj.weight::matrix0": 10.2226873776879, + "module.module.decoder.layers.55.self_attention.k_proj.weight::matrix0": 43.53261960974978, + "module.module.decoder.layers.55.self_attention.out_proj.weight::matrix0": 85.59810471745624, + "module.module.decoder.layers.55.self_attention.q_proj.weight::matrix0": 50.46476328643579, + "module.module.decoder.layers.55.self_attention.v_proj.weight::matrix0": 92.45428836506545, + "module.module.decoder.layers.55.self_attention.z_proj.weight::matrix0": 84.72566371924951, + "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight::matrix0": 73.77435728397843, + "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight::matrix0": 100.28248258639346, + "module.module.decoder.layers.6.self_attention.a_proj.weight::matrix0": 13.256157554594525, + "module.module.decoder.layers.6.self_attention.b_proj.weight::matrix0": 10.47615456064703, + "module.module.decoder.layers.6.self_attention.k_proj.weight::matrix0": 49.91207580635713, + "module.module.decoder.layers.6.self_attention.out_proj.weight::matrix0": 93.75835437052925, + "module.module.decoder.layers.6.self_attention.q_proj.weight::matrix0": 50.53962266611289, + "module.module.decoder.layers.6.self_attention.v_proj.weight::matrix0": 101.85052572323306, + "module.module.decoder.layers.6.self_attention.z_proj.weight::matrix0": 95.0867753002486, + "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight::matrix0": 72.87245482403051, + "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight::matrix0": 100.77310569310713, + "module.module.decoder.layers.7.self_attention.a_proj.weight::matrix0": 13.901022208508705, + "module.module.decoder.layers.7.self_attention.b_proj.weight::matrix0": 11.589560208236403, + "module.module.decoder.layers.7.self_attention.k_proj.weight::matrix0": 51.843481857099384, + "module.module.decoder.layers.7.self_attention.out_proj.weight::matrix0": 98.59997764057007, + "module.module.decoder.layers.7.self_attention.q_proj.weight::matrix0": 51.31987811199463, + "module.module.decoder.layers.7.self_attention.v_proj.weight::matrix0": 101.81266864535864, + "module.module.decoder.layers.7.self_attention.z_proj.weight::matrix0": 97.23413072999978, + "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight::matrix0": 71.39624802582588, + "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight::matrix0": 100.05403322113764, + "module.module.decoder.layers.8.self_attention.a_proj.weight::matrix0": 13.779114798261071, + "module.module.decoder.layers.8.self_attention.b_proj.weight::matrix0": 10.327106508419142, + "module.module.decoder.layers.8.self_attention.k_proj.weight::matrix0": 49.64791848106578, + "module.module.decoder.layers.8.self_attention.out_proj.weight::matrix0": 92.61027453172466, + "module.module.decoder.layers.8.self_attention.q_proj.weight::matrix0": 50.99149028739247, + "module.module.decoder.layers.8.self_attention.v_proj.weight::matrix0": 101.94783372948758, + "module.module.decoder.layers.8.self_attention.z_proj.weight::matrix0": 92.65020483916288, + "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight::matrix0": 71.81661842317187, + "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight::matrix0": 100.31980975725436, + "module.module.decoder.layers.9.self_attention.a_proj.weight::matrix0": 15.6454447948475, + "module.module.decoder.layers.9.self_attention.b_proj.weight::matrix0": 10.613946668754954, + "module.module.decoder.layers.9.self_attention.k_proj.weight::matrix0": 50.041912102041195, + "module.module.decoder.layers.9.self_attention.out_proj.weight::matrix0": 92.5803859704413, + "module.module.decoder.layers.9.self_attention.q_proj.weight::matrix0": 50.920117484869756, + "module.module.decoder.layers.9.self_attention.v_proj.weight::matrix0": 100.56893327743207, + "module.module.decoder.layers.9.self_attention.z_proj.weight::matrix0": 90.66030936174982, + "module.module.embedding.word_embeddings.weight::matrix0": 5267.082135929015, + "module.module.output_layer.weight::matrix0": 796.80402920877 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261010_231200-gdb9n0eb/run-gdb9n0eb.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261010_231200-gdb9n0eb/run-gdb9n0eb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d5ed874c22a3682a6de5a379cb6536d0c989d49d --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261010_231200-gdb9n0eb/run-gdb9n0eb.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cd26a458c1298bd55f0316ecd6cbadf5b4ffcecbfcccefb4d6ea91afda54b48 +size 261671 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_010425-k7i86mla/run-k7i86mla.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_010425-k7i86mla/run-k7i86mla.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6c440e9578df66c00a4508191ef1ef03d78e58d0 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_010425-k7i86mla/run-k7i86mla.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66fe3fefbac032dd56a0d68b38c7b95cd07bf20d0d9b631d05d5d5133c8c07ca +size 194319 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_013957-a2kv2ydx/run-a2kv2ydx.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_013957-a2kv2ydx/run-a2kv2ydx.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e6dd055ee2a64d5bdf211d6a13936c67c2560e79 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_013957-a2kv2ydx/run-a2kv2ydx.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc1f7312c41e7e7b708b391c7422225c96d7cfca216264b22d9c43fcb53c3a34 +size 271650 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_015603-9a1n96nr/run-9a1n96nr.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_015603-9a1n96nr/run-9a1n96nr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..145d745fde19a32c0f6f50800c3edb67bd26c403 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_015603-9a1n96nr/run-9a1n96nr.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2449bcfed64c3897b0d08086315ba366dc55cfa0c2e23b6d70b862fd7d7a23 +size 270706 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_021843-lzgz3u05/run-lzgz3u05.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_021843-lzgz3u05/run-lzgz3u05.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e240e3c29c39fa6b8abe8a8036ca82633068c8d3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_021843-lzgz3u05/run-lzgz3u05.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a745c1c3300e5a314d9b6e62d5f8763555c01bb9bae424153c379debbeef09f0 +size 275527 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_024051-df4j15aa/run-df4j15aa.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_024051-df4j15aa/run-df4j15aa.wandb new file mode 100644 index 0000000000000000000000000000000000000000..39c1923c12ad3a4aebc8bf1af4b655fa78096840 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_024051-df4j15aa/run-df4j15aa.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8c5186033db2017a5eeae80bcfe730c6341639ba2e3962be12ded88d754511 +size 10485760 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/run-acr8mx9p.wandb b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/run-acr8mx9p.wandb new file mode 100644 index 0000000000000000000000000000000000000000..137798abbb6c846e72a88fd4158228066fa577c1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/run-acr8mx9p.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93506b12d23660895e0964ecde2288692de1178b1084929c8837e9d9e1fde33b +size 403797318