diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/mlc-chat-config.json b/mlc-chat-config.json new file mode 100644 index 0000000000000000000000000000000000000000..a5246d0c19adcd4e62895d89ac9097b3aa624fd5 --- /dev/null +++ b/mlc-chat-config.json @@ -0,0 +1,91 @@ +{ + "version": "0.1.0", + "model_type": "qwen3_5", + "quantization": "q4f16_1", + "model_config": { + "hidden_size": 4096, + "intermediate_size": 12288, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "vocab_size": 248320, + "rope_theta": 10000000, + "head_dim": 256, + "hidden_act": "silu", + "attention_bias": false, + "tie_word_embeddings": false, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_conv_kernel_dim": 4, + "full_attention_interval": 4, + "partial_rotary_factor": 0.25, + "context_window_size": 4096, + "prefill_chunk_size": 2048, + "tensor_parallel_shards": 1, + "dtype": "bfloat16", + "max_batch_size": 128 + }, + "vocab_size": 248320, + "context_window_size": 4096, + "sliding_window_size": -1, + "prefill_chunk_size": 2048, + "attention_sink_size": -1, + "tensor_parallel_shards": 1, + "pipeline_parallel_stages": 1, + "active_vocab_size": 248077, + "temperature": 0.6, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "repetition_penalty": 1.0, + "top_p": 0.95, + "tokenizer_files": [ + "tokenizer.json", + "tokenizer_config.json" + ], + "tokenizer_info": { + "token_postproc_method": "byte_level", + "prepend_space_in_encode": false, + "strip_space_in_decode": false + }, + "conv_template": { + "name": "qwen3_5", + "system_template": "<|im_start|>system\n{system_message}<|im_end|>\n", + "system_message": "You are a helpful assistant.", + "system_prefix_token_ids": null, + "add_role_after_system_message": true, + "roles": { + "user": "<|im_start|>user", + "assistant": "<|im_start|>assistant\n" + }, + "role_templates": { + "user": "{user_message}", + "assistant": "{assistant_message}", + "tool": "{tool_message}" + }, + "messages": [], + "seps": [ + "<|im_end|>\n" + ], + "role_content_sep": "\n", + "role_empty_sep": "\n", + "stop_str": [ + "<|endoftext|>", + "<|im_end|>" + ], + "stop_token_ids": [ + 248046, + 248044 + ], + "strip_reasoning_in_history": false, + "function_string": "", + "use_function_calling": false + }, + "pad_token_id": 248055, + "bos_token_id": 1, + "eos_token_id": 248044, + "model_task": "chat", + "embedding_metadata": null +} diff --git a/params_shard_0.bin b/params_shard_0.bin new file mode 100644 index 0000000000000000000000000000000000000000..1f448f35e7504be249a4332c57501a2611010717 --- /dev/null +++ b/params_shard_0.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f673d6395bd3553f2720c4a7432dbe4bd052379d43a8681b294f072d98e2177 +size 508559360 diff --git a/params_shard_1.bin b/params_shard_1.bin new file mode 100644 index 0000000000000000000000000000000000000000..9e33eb60c25e572f5c915102c0f79934d290946f --- /dev/null +++ b/params_shard_1.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d3a48bd408002a0a1d743e5bd4ee2f11b8164c03dd4da17b1e95392ad82f3a +size 63569920 diff --git a/params_shard_10.bin b/params_shard_10.bin new file mode 100644 index 0000000000000000000000000000000000000000..edbcf8422c3aa303b51cb6f9aebc631195088af0 --- /dev/null +++ b/params_shard_10.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5c20a547a7e7d05726dea9cc1b8b6477aa8071188148b12cc664010e227b1e +size 50331648 diff --git a/params_shard_100.bin b/params_shard_100.bin new file mode 100644 index 0000000000000000000000000000000000000000..16b5421552ab6fac3e2e7d7efc5bfed9b4743962 --- /dev/null +++ b/params_shard_100.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e79faf4eec93dc519d66255a7a430b993f58c3685eacb612a98e4560206e36 +size 29893120 diff --git a/params_shard_101.bin b/params_shard_101.bin new file mode 100644 index 0000000000000000000000000000000000000000..870d4e893fbffdba6536a41d8e81f5f756ebaae3 --- /dev/null +++ b/params_shard_101.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30fd3323102e83e14d4dbf415122d869e84ace401b935c21968ad6c807dd8f84 +size 28533376 diff --git a/params_shard_102.bin b/params_shard_102.bin new file mode 100644 index 0000000000000000000000000000000000000000..d5d33d407ce7151e052b8fa7a587e24b57a61bb8 --- /dev/null +++ b/params_shard_102.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d040eade8e0dce8741149a25357a1feea2a0a15b66fde39482f041313dff0fe4 +size 25165824 diff --git a/params_shard_103.bin b/params_shard_103.bin new file mode 100644 index 0000000000000000000000000000000000000000..945a7e2e324b11dbd957059e329eaefbbe8b7f48 --- /dev/null +++ b/params_shard_103.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1470d40f594373f8d432cc109836cb8ea85a956c5bddd4c528fc6ccb6c06d72a +size 50331648 diff --git a/params_shard_104.bin b/params_shard_104.bin new file mode 100644 index 0000000000000000000000000000000000000000..886abc195e01bbfad474e5cb70ab928bb18906c6 --- /dev/null +++ b/params_shard_104.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c62d1dccc5ff078a8a771875881072378525365baa6dd6bbc118ec4c45812ea +size 16777216 diff --git a/params_shard_105.bin b/params_shard_105.bin new file mode 100644 index 0000000000000000000000000000000000000000..f04dedb94e0e64d800840cbca9ef57c31ed5b701 --- /dev/null +++ b/params_shard_105.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6186a66a63ae230d566cc0eec0eafaf582db380e72d2977b06ab64940cdaa979 +size 30638464 diff --git a/params_shard_106.bin b/params_shard_106.bin new file mode 100644 index 0000000000000000000000000000000000000000..76dd96d89ae44b829b236a0d2e141c8998a0f16d --- /dev/null +++ b/params_shard_106.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d6ea09e2e6c752363825c24f35c83aaae894eff0288bb818b79358e8f663b5d +size 25165824 diff --git a/params_shard_107.bin b/params_shard_107.bin new file mode 100644 index 0000000000000000000000000000000000000000..49227857fd45e856c03ae8b5d8b396789de267ed --- /dev/null +++ b/params_shard_107.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8918b71534be6417967dec4631042dace3b43eff373dc5baad7bf32523485996 +size 50331648 diff --git a/params_shard_108.bin b/params_shard_108.bin new file mode 100644 index 0000000000000000000000000000000000000000..e040e24bbca428ea2d3b30dca194dda35141854f --- /dev/null +++ b/params_shard_108.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c751c0fe0d043eb857ac135121e1549bb12537fef1aa5785117a8fdc09d15727 +size 16777216 diff --git a/params_shard_109.bin b/params_shard_109.bin new file mode 100644 index 0000000000000000000000000000000000000000..93c02d11e48d07d0ee3bc342bfd33a93ab7765c4 --- /dev/null +++ b/params_shard_109.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5ee27f2efd5a4ddb4663dedd577356e35451b54f1659d38f3d583d79c8d18eb +size 30638464 diff --git a/params_shard_11.bin b/params_shard_11.bin new file mode 100644 index 0000000000000000000000000000000000000000..200daf7e59646d9a93a506a0b9055a37babb222d --- /dev/null +++ b/params_shard_11.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91f3e79588a56689f309a82770b123c80f9d0bfa7fd0c683c956f5967b804043 +size 16777216 diff --git a/params_shard_110.bin b/params_shard_110.bin new file mode 100644 index 0000000000000000000000000000000000000000..1b303352887e9c75c022f53f392bea1c4a986d46 --- /dev/null +++ b/params_shard_110.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9ba187e982f64e5e703a12763a76443b85be77427dfbc37c09c6503e50c89ff +size 25165824 diff --git a/params_shard_111.bin b/params_shard_111.bin new file mode 100644 index 0000000000000000000000000000000000000000..571293b2fc41bddd3b73fc9e79bb92ede6a22ce3 --- /dev/null +++ b/params_shard_111.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:757328f8401e8732969c69d0d002a7fa326ab5195a77eff07be07e58928e03fe +size 50331648 diff --git a/params_shard_112.bin b/params_shard_112.bin new file mode 100644 index 0000000000000000000000000000000000000000..28c88d69ac713215afe3f2b01bfc0e80751426ed --- /dev/null +++ b/params_shard_112.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a4e3188b5ceab4d31f5b4de8458dfde9028053b032d8a2ba27005446ae1c3a +size 25165824 diff --git a/params_shard_113.bin b/params_shard_113.bin new file mode 100644 index 0000000000000000000000000000000000000000..4d56a4e8499e020f353f8693a340069207cee7e9 --- /dev/null +++ b/params_shard_113.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b39451a4aef14b822ea2cee9a102a6d3eb3679c155bde115ec22852cf0a9d3c +size 50331648 diff --git a/params_shard_114.bin b/params_shard_114.bin new file mode 100644 index 0000000000000000000000000000000000000000..835eb7e875a8734632ce6a2127ca61b6532f95e4 --- /dev/null +++ b/params_shard_114.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db45d8913d7b020614fe9de71159612b0ef23a393531710e7c0f8c69508436e9 +size 31473920 diff --git a/params_shard_115.bin b/params_shard_115.bin new file mode 100644 index 0000000000000000000000000000000000000000..24934bce382b69678c35a7d4b555a72cec8e3e57 --- /dev/null +++ b/params_shard_115.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75fb9de4984ec9bd26c35e50a0c7e812c3f5051b8ac521c15ec44fa957e08984 +size 29893120 diff --git a/params_shard_116.bin b/params_shard_116.bin new file mode 100644 index 0000000000000000000000000000000000000000..78032338405e7a7c2b3f9035d4fa5853dd290266 --- /dev/null +++ b/params_shard_116.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:944375a6217844d1c89224a02381a53e70505bc6832d1396e793317c0bc6a18b +size 28533376 diff --git a/params_shard_117.bin b/params_shard_117.bin new file mode 100644 index 0000000000000000000000000000000000000000..71e9a0e272ea38625751719bd0754fc263301125 --- /dev/null +++ b/params_shard_117.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba0ee9883e76dc68a2ea158b61539a86477345e4374ba17862d63d6603074369 +size 25165824 diff --git a/params_shard_118.bin b/params_shard_118.bin new file mode 100644 index 0000000000000000000000000000000000000000..af85934468af4cd90a3d054da6eeb94ac3d184a7 --- /dev/null +++ b/params_shard_118.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a77868bafce30637fcddf5e9bd4e4471187aef37b506cde88909f5cad41b013d +size 50331648 diff --git a/params_shard_119.bin b/params_shard_119.bin new file mode 100644 index 0000000000000000000000000000000000000000..4765066213563d173a1b21f1249cfd201e82f982 --- /dev/null +++ b/params_shard_119.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a14a4af22990eee0c709df4a6c8fa5f2ca3919d0f7148e0743c9ddf15a747475 +size 16777216 diff --git a/params_shard_12.bin b/params_shard_12.bin new file mode 100644 index 0000000000000000000000000000000000000000..9ed4b6d7bb4fa6e859108af7bf56c21f57a4c2fe --- /dev/null +++ b/params_shard_12.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b3cbfcbd82b02350899458a4c60948aab9a7979f8e16c46afc1ce2fa12e023d +size 30638464 diff --git a/params_shard_120.bin b/params_shard_120.bin new file mode 100644 index 0000000000000000000000000000000000000000..7f3e8aefdbea38805f11e22a5ef92ddf456b8051 --- /dev/null +++ b/params_shard_120.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efe9d94d56433629c5a238e8534e8d54343d34ac63f9aa8d703e3c0a080b7240 +size 30638464 diff --git a/params_shard_121.bin b/params_shard_121.bin new file mode 100644 index 0000000000000000000000000000000000000000..2fd35670123f58864cf96a63e6b37edd7e8d6e6e --- /dev/null +++ b/params_shard_121.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8156205631e2d04fde5e104c30e4578e494066eea5ba5e50f2072b434a5f68be +size 25165824 diff --git a/params_shard_122.bin b/params_shard_122.bin new file mode 100644 index 0000000000000000000000000000000000000000..ef21fc2547d9e5d85ed14b93fdf3887e36b8f2d7 --- /dev/null +++ b/params_shard_122.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dda609a893dc3b0e5c5c06dd07358cd080d3887e0f64806edce25094f2b0539 +size 50331648 diff --git a/params_shard_123.bin b/params_shard_123.bin new file mode 100644 index 0000000000000000000000000000000000000000..85f45c78dcf6f949a7b0483f901e677f44432c51 --- /dev/null +++ b/params_shard_123.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db84cf769db1a8bb3ae3d476cac83c012236319eb05c014246f39e03181e61c7 +size 28328192 diff --git a/params_shard_13.bin b/params_shard_13.bin new file mode 100644 index 0000000000000000000000000000000000000000..64251d6c370cf5e0cf45ffdd089a66ac09afa3d3 --- /dev/null +++ b/params_shard_13.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b993227bb28e2c77d65472a88474dd446800aaa7e65b841eff73ff31e54ff797 +size 25165824 diff --git a/params_shard_14.bin b/params_shard_14.bin new file mode 100644 index 0000000000000000000000000000000000000000..a3e9c5d4b1e47231a214d79cf90edeae8ad8b5f6 --- /dev/null +++ b/params_shard_14.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:022d0daf933f43718030fb2b8ec4115acc5d94e0d9a466aa755385e63da460ba +size 50331648 diff --git a/params_shard_15.bin b/params_shard_15.bin new file mode 100644 index 0000000000000000000000000000000000000000..1a063e260f898ddc456f940afc26f5b814690910 --- /dev/null +++ b/params_shard_15.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eddb6113dcdc761781419433b80dffe0ced908e042396935858fd851c4db3b5 +size 25165824 diff --git a/params_shard_16.bin b/params_shard_16.bin new file mode 100644 index 0000000000000000000000000000000000000000..c33ee340921c6f8b20886031c23d313e217101f8 --- /dev/null +++ b/params_shard_16.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0ad92a315885fafc1be118c549a2d1156c14e0a0368ad5efb418c1c3a42dc47 +size 50331648 diff --git a/params_shard_17.bin b/params_shard_17.bin new file mode 100644 index 0000000000000000000000000000000000000000..c5444f952ea6e69260a93d5070ae23960eb4a719 --- /dev/null +++ b/params_shard_17.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f29d31348d9fbd456d7eafda08183ee7de6f7a502afcba0de7b287e0af5801e +size 20971520 diff --git a/params_shard_18.bin b/params_shard_18.bin new file mode 100644 index 0000000000000000000000000000000000000000..0d5b7e32741a1644e10d51130ccf089372569246 --- /dev/null +++ b/params_shard_18.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d08ea1ea878b9ffa8942c78372e7e873165640b46a36f954e894605541ac511 +size 30958080 diff --git a/params_shard_19.bin b/params_shard_19.bin new file mode 100644 index 0000000000000000000000000000000000000000..36e2823c47042d683e5a8bc509b1dcd69092dd08 --- /dev/null +++ b/params_shard_19.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2381ec220af5a2a72793d5a2bd616cec49acc97f380aefa49f28fa6fd5cee2e2 +size 28533376 diff --git a/params_shard_2.bin b/params_shard_2.bin new file mode 100644 index 0000000000000000000000000000000000000000..9f3a9ec0c9f76f4faeb34cdfe6ebfc5a5163eef9 --- /dev/null +++ b/params_shard_2.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f1f5ba3c537d5d905f20eb3ef34ea796088943d64dbc275725d2a80ec78278f +size 508559360 diff --git a/params_shard_20.bin b/params_shard_20.bin new file mode 100644 index 0000000000000000000000000000000000000000..2c2d54ea113ab7dfb83904d0cb401de5ed3f9498 --- /dev/null +++ b/params_shard_20.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb3e5a30b299622e68ae7480a5d0e183d60cd4a43310ebb61cfb53962f16c45a +size 25165824 diff --git a/params_shard_21.bin b/params_shard_21.bin new file mode 100644 index 0000000000000000000000000000000000000000..35f549ea1d08e1c03d0ac19afbff409bd28f8037 --- /dev/null +++ b/params_shard_21.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a16e365465180f30083c433da83fd7144f192df1c1494e92c6b6cdad202eedfb +size 50331648 diff --git a/params_shard_22.bin b/params_shard_22.bin new file mode 100644 index 0000000000000000000000000000000000000000..b9bad82fb97d67ba46cbf66619076c0be82e09b4 --- /dev/null +++ b/params_shard_22.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:175663cded0eb783d54eff02b186c245a319c1418ac6d425f37af27446b5b91e +size 16777216 diff --git a/params_shard_23.bin b/params_shard_23.bin new file mode 100644 index 0000000000000000000000000000000000000000..70b3a796f6396a405ec26bac918e342dbc4351b0 --- /dev/null +++ b/params_shard_23.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bc0b21c0a82a193b8ffa50e8822416fa47d03069a1485039625c1c66ad8a852 +size 30638464 diff --git a/params_shard_24.bin b/params_shard_24.bin new file mode 100644 index 0000000000000000000000000000000000000000..a946c17825199c7cd01cb48d588e83036420b447 --- /dev/null +++ b/params_shard_24.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94b3c67a8f6f5c39cfa5318acbe33e2b261eab0fba480fedb448250984096af +size 25165824 diff --git a/params_shard_25.bin b/params_shard_25.bin new file mode 100644 index 0000000000000000000000000000000000000000..b9c9f25a9ac6edc92768e1d430137b08aead855b --- /dev/null +++ b/params_shard_25.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f10f169aa277bd1f7eeaec528869df114b49aa64d93b63bc41522e6a8ee686c +size 50331648 diff --git a/params_shard_26.bin b/params_shard_26.bin new file mode 100644 index 0000000000000000000000000000000000000000..5c9bfec59c9d5407c841ea8c420c0769a51fec03 --- /dev/null +++ b/params_shard_26.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d99f958eb935e4728008520e07de6f3fa6f0aa183527c1d85a9eed96f0d32d +size 16777216 diff --git a/params_shard_27.bin b/params_shard_27.bin new file mode 100644 index 0000000000000000000000000000000000000000..09141d28332072aa7a41719323d4e0b4685f4b3b --- /dev/null +++ b/params_shard_27.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2be407046709884b615437f869eeb8b14a65d5e79892cd79907494c997038a23 +size 30638464 diff --git a/params_shard_28.bin b/params_shard_28.bin new file mode 100644 index 0000000000000000000000000000000000000000..104d8be9e564819ca9ba210012aa34164da0a33f --- /dev/null +++ b/params_shard_28.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406486e786266913f1fb10dfd65f0fabe4d5f405af1289bac673e43ce030050a +size 25165824 diff --git a/params_shard_29.bin b/params_shard_29.bin new file mode 100644 index 0000000000000000000000000000000000000000..72b922d5f15a9cee907186c85346671b92ffe4af --- /dev/null +++ b/params_shard_29.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef7c590656b998dd5954005e5fa5973e2b7de64e7da6fed91669a7a7b7f8b68 +size 50331648 diff --git a/params_shard_3.bin b/params_shard_3.bin new file mode 100644 index 0000000000000000000000000000000000000000..38bf71b4d723b27133eee44cd399886b24b716da --- /dev/null +++ b/params_shard_3.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f0f4b81a45cbb2034021f72ad8fa9972a868d38447cd21bf668dd97627cd023 +size 63569920 diff --git a/params_shard_30.bin b/params_shard_30.bin new file mode 100644 index 0000000000000000000000000000000000000000..3deb05ac2140b3ff629a8a34359551883e2580ff --- /dev/null +++ b/params_shard_30.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605ee7ff47d07a6e0b496d04b87c6a513efe3c13e7f23c1be7b4e34e2a306150 +size 25165824 diff --git a/params_shard_31.bin b/params_shard_31.bin new file mode 100644 index 0000000000000000000000000000000000000000..9cfd0f344250bb23894629ab49970622f764a004 --- /dev/null +++ b/params_shard_31.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b63ae606c4f2e76e974fd273127b5cc1da0e4f49c2fa6c490239cb4ce00c7d65 +size 50331648 diff --git a/params_shard_32.bin b/params_shard_32.bin new file mode 100644 index 0000000000000000000000000000000000000000..20bd2ac3ab0492b86a1ce228a0035917fabd2f90 --- /dev/null +++ b/params_shard_32.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95b243b46ecc7f3d9d0b6e59daaaa5f0769de9b8328019a9a2b71097b1326d24 +size 31473920 diff --git a/params_shard_33.bin b/params_shard_33.bin new file mode 100644 index 0000000000000000000000000000000000000000..a303671e6fc01209388c4357ffd16da5fedefd46 --- /dev/null +++ b/params_shard_33.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb7f50636ee3f75a6524b2fc4981e19467520c5d946fe25f7318b301ab6eb072 +size 29893120 diff --git a/params_shard_34.bin b/params_shard_34.bin new file mode 100644 index 0000000000000000000000000000000000000000..73a442ea2ab589d8d4107c768229e19ee48df331 --- /dev/null +++ b/params_shard_34.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05efe938226289d71752e23e751d93ef56b5142ae5659fe807ced176a93c0443 +size 28533376 diff --git a/params_shard_35.bin b/params_shard_35.bin new file mode 100644 index 0000000000000000000000000000000000000000..a1da9bd8a82dab445e22d84de3c561876b0a3ad5 --- /dev/null +++ b/params_shard_35.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9762c1a0cea063fc5c507a9a44555f6ac00742243ab3750886ad734f5eac0b90 +size 25165824 diff --git a/params_shard_36.bin b/params_shard_36.bin new file mode 100644 index 0000000000000000000000000000000000000000..175e7cc396bbe03225e603826b7a7a841b141081 --- /dev/null +++ b/params_shard_36.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0f1e9b3c32babfd58cd0b048359996029f0f74e975ba765f4bcac375045b958 +size 50331648 diff --git a/params_shard_37.bin b/params_shard_37.bin new file mode 100644 index 0000000000000000000000000000000000000000..1bf36ef219bfd814854c6ba0160eaae7cc32983c --- /dev/null +++ b/params_shard_37.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8339a2646c8e041c6c65b02f0aa6747037886cfa9c32ec4f26d1071752fe1423 +size 16777216 diff --git a/params_shard_38.bin b/params_shard_38.bin new file mode 100644 index 0000000000000000000000000000000000000000..530c60651de11e17b9a71076e35697c0af665d5b --- /dev/null +++ b/params_shard_38.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:132cc03ac20b2242ca48dd49b08b2e469fe4a44c291bd01ca2586b3a7cb2f092 +size 30638464 diff --git a/params_shard_39.bin b/params_shard_39.bin new file mode 100644 index 0000000000000000000000000000000000000000..7ea95d9ebd54c4cdcbac8c33b4cfbf96d6cc0460 --- /dev/null +++ b/params_shard_39.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71a6e8e32776c4c852e3c663036f86057f7192f4d7fd5224454c187b39b5b189 +size 25165824 diff --git a/params_shard_4.bin b/params_shard_4.bin new file mode 100644 index 0000000000000000000000000000000000000000..3a52b79f7c866e43d9510783a04cb88ef662c2e8 --- /dev/null +++ b/params_shard_4.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97d989511324d3d22ca706b7916f2b5e3d3dcf25eed8ff6d8b518b009cdbdc0d +size 28533120 diff --git a/params_shard_40.bin b/params_shard_40.bin new file mode 100644 index 0000000000000000000000000000000000000000..4a1dd2053d427202c0f1bc336400bce3a6b9de30 --- /dev/null +++ b/params_shard_40.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d5125c75f96c7370edc69f7a5008399622bb6fd668e3fa7aa18736c1f1e953b +size 50331648 diff --git a/params_shard_41.bin b/params_shard_41.bin new file mode 100644 index 0000000000000000000000000000000000000000..8e9a51f8048762cf1634f88a233ac7b241870414 --- /dev/null +++ b/params_shard_41.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36d1dff99382b5bed930b305152ab37492f6fa291735e0843b6742bdc2c9f36d +size 16777216 diff --git a/params_shard_42.bin b/params_shard_42.bin new file mode 100644 index 0000000000000000000000000000000000000000..db51677ad53e7f4a437297888df43c94948ca6b6 --- /dev/null +++ b/params_shard_42.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b8604f80f90cd9a65ceb267afe0345bfec5070f99d19de875bf49dc99a8ac32 +size 30638464 diff --git a/params_shard_43.bin b/params_shard_43.bin new file mode 100644 index 0000000000000000000000000000000000000000..e281d15d8e86c0edf5c022d47c62476049d79099 --- /dev/null +++ b/params_shard_43.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5873f24c1738b1206332c4726bfbb497cd30e38ab5d8a50019e0bc02b9a98df2 +size 25165824 diff --git a/params_shard_44.bin b/params_shard_44.bin new file mode 100644 index 0000000000000000000000000000000000000000..72c4b51b7f5c220aced2b4bef570c56667b0252b --- /dev/null +++ b/params_shard_44.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1c872a9833a27c521f4d51ffee0968b207a85ae03497f1e692540905aa8aac7 +size 50331648 diff --git a/params_shard_45.bin b/params_shard_45.bin new file mode 100644 index 0000000000000000000000000000000000000000..257bb673f1042e59d903d1278fff0a887bced956 --- /dev/null +++ b/params_shard_45.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81cee397b34ae88059b46fff92a4737c1ac9f64d5cef79e1ab99c11aa7c4b87f +size 25165824 diff --git a/params_shard_46.bin b/params_shard_46.bin new file mode 100644 index 0000000000000000000000000000000000000000..1574de02957c8fc72c390e62f7361449443a7c2a --- /dev/null +++ b/params_shard_46.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29260cdd8c70c063afb3836d35e5a547cf68f3c5fee611f1a7e0f967d0ecf05b +size 50331648 diff --git a/params_shard_47.bin b/params_shard_47.bin new file mode 100644 index 0000000000000000000000000000000000000000..d43b1ded01b9012d0f6a70996c9b6876f9961551 --- /dev/null +++ b/params_shard_47.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8b1fc3adb9777d5703cd945dd5c8edc605b5b7da208b6f8176ddb639ac642b1 +size 31473920 diff --git a/params_shard_48.bin b/params_shard_48.bin new file mode 100644 index 0000000000000000000000000000000000000000..ece454d6a42304b5c8f85e52ac64709ebff2d332 --- /dev/null +++ b/params_shard_48.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aea32fe9815265f9355a1ca73268cd371891ff4a0fbec9d0c597aed08a14097e +size 29893120 diff --git a/params_shard_49.bin b/params_shard_49.bin new file mode 100644 index 0000000000000000000000000000000000000000..e3461668ccfd7647d469be3c9743ef96dc617339 --- /dev/null +++ b/params_shard_49.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76c65ee92b9c27527b4ea062aa8c2163eba2f5faff47e5bb6eeeddb09f78a645 +size 28533376 diff --git a/params_shard_5.bin b/params_shard_5.bin new file mode 100644 index 0000000000000000000000000000000000000000..6cd67a65ecfb98d1d49cc41a93c370ce7d5de6fd --- /dev/null +++ b/params_shard_5.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65bdfc4c33766a43480a181c37c7993c81063edb2fa6c3ac8ade8ce1e22343b3 +size 25165824 diff --git a/params_shard_50.bin b/params_shard_50.bin new file mode 100644 index 0000000000000000000000000000000000000000..55329d315a49cc23a632592693f34b11ceedc15d --- /dev/null +++ b/params_shard_50.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bacd52b1e9d2bee318655cfc2d532db5f8ff814877cce8d1f5e5a1411a2184f +size 25165824 diff --git a/params_shard_51.bin b/params_shard_51.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce8ffc3e569a30761787211e1e0641fc07fedf3c --- /dev/null +++ b/params_shard_51.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c6317c60b27b4f37c490e17b1971c475c56b4e95c88e21e7e84261eb13d2da0 +size 50331648 diff --git a/params_shard_52.bin b/params_shard_52.bin new file mode 100644 index 0000000000000000000000000000000000000000..1135a13d86d2d74fa39530c2a64b4e016518d4cc --- /dev/null +++ b/params_shard_52.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:771835fe39ddddec5319db7789374b02e8ba1f9bc79f53da97ccf5677051737a +size 16777216 diff --git a/params_shard_53.bin b/params_shard_53.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8ec2ffe001551f620cdc16bf667f29375da2510 --- /dev/null +++ b/params_shard_53.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d3cffbc8ed1dab4924e52b466ea18f1c39c108900c1aea925263c125bf34be +size 30638464 diff --git a/params_shard_54.bin b/params_shard_54.bin new file mode 100644 index 0000000000000000000000000000000000000000..61edaaacf0bb4e380fb284021d2c3415352e479a --- /dev/null +++ b/params_shard_54.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40001cc1ad01e69e73da212124ecc551a41eaff676ba39fe2bbd500cafabe27e +size 25165824 diff --git a/params_shard_55.bin b/params_shard_55.bin new file mode 100644 index 0000000000000000000000000000000000000000..ea46ee8dc1148baf6c5661b5691bcb9dd63f7d04 --- /dev/null +++ b/params_shard_55.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19bb3ee351fce2dd97116189877fa1c560b5ecff08e4b96f231ac3b2b2ee7da3 +size 50331648 diff --git a/params_shard_56.bin b/params_shard_56.bin new file mode 100644 index 0000000000000000000000000000000000000000..9e200f10a6478aa285b44de6d4c3976e8c484ec2 --- /dev/null +++ b/params_shard_56.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c11b12b71614f654fad804f379c0f11636eb350e933e488fada6380ac7cea7a3 +size 16777216 diff --git a/params_shard_57.bin b/params_shard_57.bin new file mode 100644 index 0000000000000000000000000000000000000000..fdf4d035f24202e4b3d09baa7b0c23d1adafae91 --- /dev/null +++ b/params_shard_57.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3baf173c43089408271d4a2f830c971b7949a3023e79c9ad35f6de1f92b01604 +size 30638464 diff --git a/params_shard_58.bin b/params_shard_58.bin new file mode 100644 index 0000000000000000000000000000000000000000..3b3d683aabc889e1c6cc99d1f7c1266098228d24 --- /dev/null +++ b/params_shard_58.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e190b4c76359b8103dda4a202b903b9511b30c47b9cd66c57cf8b276a1967588 +size 25165824 diff --git a/params_shard_59.bin b/params_shard_59.bin new file mode 100644 index 0000000000000000000000000000000000000000..5b4aa6003ca36e222a3b71919b7ba7e310624025 --- /dev/null +++ b/params_shard_59.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13c334860e12411e2ab2eb4c7acc0c6ab4f1f7a95843760042a27b42066acce1 +size 50331648 diff --git a/params_shard_6.bin b/params_shard_6.bin new file mode 100644 index 0000000000000000000000000000000000000000..1b6277542bb4b57714240c84a4183c67dda3c079 --- /dev/null +++ b/params_shard_6.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75dae299069f2fada980645c29e88f33c8f8c817ba08920ecb91c8f79d7df87a +size 50331648 diff --git a/params_shard_60.bin b/params_shard_60.bin new file mode 100644 index 0000000000000000000000000000000000000000..2eff99e86f72b35f2cdcc93b542bd95878f6d7ae --- /dev/null +++ b/params_shard_60.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3784c4fa0850c62c02a00e745ac2e32c736aa02e1881e605bf1f0e27f8d89ec +size 16777216 diff --git a/params_shard_61.bin b/params_shard_61.bin new file mode 100644 index 0000000000000000000000000000000000000000..f7ca0ff19984100ace8646afccd5e4b7f6807b3b --- /dev/null +++ b/params_shard_61.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17199f70bb4aa7205c2358753eec19e3883b9618b0831481826fea185acf5d2c +size 30638464 diff --git a/params_shard_62.bin b/params_shard_62.bin new file mode 100644 index 0000000000000000000000000000000000000000..3f5e09bb43a17ff509959a570c5db069a995ae63 --- /dev/null +++ b/params_shard_62.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d6e53c36df1c759e8cc2c9a284cadfb8a9d45694af8f8198f9c31df59d965f +size 25165824 diff --git a/params_shard_63.bin b/params_shard_63.bin new file mode 100644 index 0000000000000000000000000000000000000000..e564dafcfdd01e319aef3bbdf5f2a0ae07332f27 --- /dev/null +++ b/params_shard_63.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:677f77ba2847bb4d2ee40dcff226e23afa719f5a6d251c18b7aa8e1bc3122ef7 +size 50331648 diff --git a/params_shard_64.bin b/params_shard_64.bin new file mode 100644 index 0000000000000000000000000000000000000000..7e4d03e631c724a7e741b9839fa5d443d93305a4 --- /dev/null +++ b/params_shard_64.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33a9ef71561035804e546a04b4928f3292eb1af8d42fcff343d1799fb53430db +size 25165824 diff --git a/params_shard_65.bin b/params_shard_65.bin new file mode 100644 index 0000000000000000000000000000000000000000..ced41bb5e0a5f5a890164e15921ef2106a6992ab --- /dev/null +++ b/params_shard_65.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1befff9a43672e80ce9e7f6c177f0c8ce1cfe5c1893cad6fdbdf9f530a5a2bc9 +size 50331648 diff --git a/params_shard_66.bin b/params_shard_66.bin new file mode 100644 index 0000000000000000000000000000000000000000..020d0bcd7691732f2909ede18cd35c7a8814c925 --- /dev/null +++ b/params_shard_66.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:492d6947e0d58f5903b4097df2f6a6392ec80e379e9aef28d128089051b55304 +size 31473920 diff --git a/params_shard_67.bin b/params_shard_67.bin new file mode 100644 index 0000000000000000000000000000000000000000..1b5a037564d5186f02261304b0a315eb0838a078 --- /dev/null +++ b/params_shard_67.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4627005751e4db1989d6d5169eba548c5231ed43d742e05f6680ff04dea88550 +size 29893120 diff --git a/params_shard_68.bin b/params_shard_68.bin new file mode 100644 index 0000000000000000000000000000000000000000..52f6cb00c958612ece1da3750e6e76c72ffd5eec --- /dev/null +++ b/params_shard_68.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32c84e4cb91e3361c096ccb6b08a0c5c09bfec4f6eeac1920c8577d918268e31 +size 28533376 diff --git a/params_shard_69.bin b/params_shard_69.bin new file mode 100644 index 0000000000000000000000000000000000000000..cf345b4b7559b7b24c92a050d0c0f6b94e39e439 --- /dev/null +++ b/params_shard_69.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac9a5a27c4b47568fc13d023b104bfb4115355d1476fe71ef233f48b5f83e597 +size 25165824 diff --git a/params_shard_7.bin b/params_shard_7.bin new file mode 100644 index 0000000000000000000000000000000000000000..1dc81c0be2c4fed433cf9caa988571fca3f7de68 --- /dev/null +++ b/params_shard_7.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a4a5a2ff06ab98b2f2d211a2ce5e8cb4092d9f187444dd234a8d817a0b44e1 +size 16777216 diff --git a/params_shard_70.bin b/params_shard_70.bin new file mode 100644 index 0000000000000000000000000000000000000000..c3d7cf60d95ce8d582b30cda4d787c4cb311e474 --- /dev/null +++ b/params_shard_70.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f04b4b90c86fda953c2c89c092700ce0352193a6e85a0c3f8d5101126eb58a1 +size 50331648 diff --git a/params_shard_71.bin b/params_shard_71.bin new file mode 100644 index 0000000000000000000000000000000000000000..6d25b7f2d4fff3c67a6d888d4fe4629068359abe --- /dev/null +++ b/params_shard_71.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c8adf62b8a3dafb689eb7388db4bb0dc660cfb592a78de9cf029d6cc099c107 +size 16777216 diff --git a/params_shard_72.bin b/params_shard_72.bin new file mode 100644 index 0000000000000000000000000000000000000000..40faa6020436e0d89712d088aa2839833447b269 --- /dev/null +++ b/params_shard_72.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:084a16735ce5803e90689f72abd2728ca884833a536aacceaf083b19ee922892 +size 30638464 diff --git a/params_shard_73.bin b/params_shard_73.bin new file mode 100644 index 0000000000000000000000000000000000000000..477f24f6eee6781f4c9219e97714d44532c70fda --- /dev/null +++ b/params_shard_73.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:effeeccdff9a3112f60b4d7eb2fdfe935ef681d609addaa2be2ad85057d7dde9 +size 25165824 diff --git a/params_shard_74.bin b/params_shard_74.bin new file mode 100644 index 0000000000000000000000000000000000000000..ef0cfce494ed6e79d966e88efa26bd6db514be93 --- /dev/null +++ b/params_shard_74.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88f852047cc5bb0bd3a2dbd9eb3a0e34ed01806ecf52ba5e4e729726ae8932b5 +size 50331648 diff --git a/params_shard_75.bin b/params_shard_75.bin new file mode 100644 index 0000000000000000000000000000000000000000..b9c2c7298308f63349600899929b9e9482887bf8 --- /dev/null +++ b/params_shard_75.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5358e75fbe2b07703640f80ac3bfaf315ee136acd1b8fd22bc2dfd03a1ea9c86 +size 16777216 diff --git a/params_shard_76.bin b/params_shard_76.bin new file mode 100644 index 0000000000000000000000000000000000000000..8a0eadd17578966ddf68abce89a65e7f3f5deea1 --- /dev/null +++ b/params_shard_76.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7727bb3b02db2b8c8f96aeb9ef6ecdbce1d6317a46327d1b1c308435f164343 +size 30638464 diff --git a/params_shard_77.bin b/params_shard_77.bin new file mode 100644 index 0000000000000000000000000000000000000000..1895d1f8a2f6b83b49aad58626c1fbda178cd8e8 --- /dev/null +++ b/params_shard_77.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214d97da1475e507d4e54e8900f9c706ee3ee0dcac2756372ab14e16f4e46258 +size 25165824 diff --git a/params_shard_78.bin b/params_shard_78.bin new file mode 100644 index 0000000000000000000000000000000000000000..e0b3025cc899027da40fdc57ee81e70df6bf2530 --- /dev/null +++ b/params_shard_78.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e143c1be4f163a98f0f4d8d3d7712c1e463f9f989106588fd383ceee32728239 +size 50331648 diff --git a/params_shard_79.bin b/params_shard_79.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d2caa0ef94e7dc95b0d05685de2c680b68cfb39 --- /dev/null +++ b/params_shard_79.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:628c99a3d15cd634fa251fbb78c2da0c8daa6bd83bd18e917b6d72eea08536ba +size 25165824 diff --git a/params_shard_8.bin b/params_shard_8.bin new file mode 100644 index 0000000000000000000000000000000000000000..90fe4062017bbad91c85b74ebce4db1bdbee5e5a --- /dev/null +++ b/params_shard_8.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0bf4d66e1aa3d492c63ae1e5b9b199b5c674739a3fc8e2eacd93ace15fbf506 +size 30638464 diff --git a/params_shard_80.bin b/params_shard_80.bin new file mode 100644 index 0000000000000000000000000000000000000000..97708765205580df2696d16a12833d9bec070a21 --- /dev/null +++ b/params_shard_80.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f9acd20c8f9640068e35abfdd8c5b184bf9e600c16c1e6805e237273bf243d6 +size 50331648 diff --git a/params_shard_81.bin b/params_shard_81.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8e9749046812811252bdc2dbdadb2ffcb7f0999 --- /dev/null +++ b/params_shard_81.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0b2ce405a0a3f2cc087d9d08377322d1315ed98571f7058623376e24c594e64 +size 31473920 diff --git a/params_shard_82.bin b/params_shard_82.bin new file mode 100644 index 0000000000000000000000000000000000000000..8633170833175be233abae73fb3143c55717a350 --- /dev/null +++ b/params_shard_82.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d19f583034a6e109c154fbf85906553f87ae6f7316736d561fd0d34291069ab8 +size 29893120 diff --git a/params_shard_83.bin b/params_shard_83.bin new file mode 100644 index 0000000000000000000000000000000000000000..9784428e589548a440d8b813c1041440ad13b85f --- /dev/null +++ b/params_shard_83.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42dfb0e89253428bd80dfd56465205f7396f03da4895e06c4b7295de2e0b0970 +size 28533376 diff --git a/params_shard_84.bin b/params_shard_84.bin new file mode 100644 index 0000000000000000000000000000000000000000..e8e1b4b68533bcb8fe8439817f5203f5747eda35 --- /dev/null +++ b/params_shard_84.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d0ccc3c2ea12f547492bb7599a0c8b2fe4abe631ab2bd0cf84e03f0d954214 +size 25165824 diff --git a/params_shard_85.bin b/params_shard_85.bin new file mode 100644 index 0000000000000000000000000000000000000000..8a97afc7401968285079514c534a0d6a017d3e0d --- /dev/null +++ b/params_shard_85.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56b4ebf2d093468a81ac2039667a97dfb22ef214686c57ab8ac52e97f36db360 +size 50331648 diff --git a/params_shard_86.bin b/params_shard_86.bin new file mode 100644 index 0000000000000000000000000000000000000000..e1d4f2876aa8fa5b3322a33bce4455b28b77a33b --- /dev/null +++ b/params_shard_86.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9e9ddfce08611e439e1b2f65728d5ded2fafc9eec94f8af5095a991cd8378d +size 16777216 diff --git a/params_shard_87.bin b/params_shard_87.bin new file mode 100644 index 0000000000000000000000000000000000000000..df5252ebb32e79d7198d24b17a20366f95d1e631 --- /dev/null +++ b/params_shard_87.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0598611a4379d1291459683e7b8961878c08cb02c3c3f3571d6615477b7b776 +size 30638464 diff --git a/params_shard_88.bin b/params_shard_88.bin new file mode 100644 index 0000000000000000000000000000000000000000..fe93c1174281c6b39108d9a6b6532aa8c21bd1b4 --- /dev/null +++ b/params_shard_88.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610f869d69a91c1376929e0ff0506972228e1b132009db86ced630fb1983b075 +size 25165824 diff --git a/params_shard_89.bin b/params_shard_89.bin new file mode 100644 index 0000000000000000000000000000000000000000..74cf0ac26c6dfe312c822b118ed114a221bc18c0 --- /dev/null +++ b/params_shard_89.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61b530310bc4bf3a2b1797eba245fd5574717263ef82588e788107731bd97d25 +size 50331648 diff --git a/params_shard_9.bin b/params_shard_9.bin new file mode 100644 index 0000000000000000000000000000000000000000..a9cb077b24945a30b3ce74279a4bbc529aa82b83 --- /dev/null +++ b/params_shard_9.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3b4026038c8edfedb51bf69603c2568edf3e77dd05df7584276573306249b4 +size 25165824 diff --git a/params_shard_90.bin b/params_shard_90.bin new file mode 100644 index 0000000000000000000000000000000000000000..24ffc2c30753ffdbb9e51122d32234f5f7fcd236 --- /dev/null +++ b/params_shard_90.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bfe660722e9ba04ccd965f82811f547accf874c0b469bf1d9d499faaf4e7987 +size 25165824 diff --git a/params_shard_91.bin b/params_shard_91.bin new file mode 100644 index 0000000000000000000000000000000000000000..3919492cccc7b1cf88f5e676f97265bccc415fd3 --- /dev/null +++ b/params_shard_91.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb63e6859ffc947e8b915409095059574212d7a3b7f988002e7bcc60d5cbdfd1 +size 50331648 diff --git a/params_shard_92.bin b/params_shard_92.bin new file mode 100644 index 0000000000000000000000000000000000000000..69c706a9dcfb1a8a8a767eabfb2fa0e1b3fcf355 --- /dev/null +++ b/params_shard_92.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c7d6ad13c8425632c02eddc1746c8980383e8b7600b2d921dffe7ae0a9f225b +size 31473920 diff --git a/params_shard_93.bin b/params_shard_93.bin new file mode 100644 index 0000000000000000000000000000000000000000..e8618232256562aa2a3ecda51fbdc0cb8c0a3b8b --- /dev/null +++ b/params_shard_93.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7589f4b32690b9e7c19d436f29e2fa16395f8048768d45eea3c917050e955d +size 29893120 diff --git a/params_shard_94.bin b/params_shard_94.bin new file mode 100644 index 0000000000000000000000000000000000000000..43578bb1de6480f7383d22ee332be3229645f688 --- /dev/null +++ b/params_shard_94.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1d8ca6b8ceb75395d9e73252331c37217991fa2cf1a0c1f08f7765a1a7420b3 +size 28533376 diff --git a/params_shard_95.bin b/params_shard_95.bin new file mode 100644 index 0000000000000000000000000000000000000000..65cc3c67d0b8b2c00b49352e71139438def9b20f --- /dev/null +++ b/params_shard_95.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dcaf4b51fc65bbdef7168c61ef1f39a19703a66a1667ffea0a7f75a2bd93591 +size 25165824 diff --git a/params_shard_96.bin b/params_shard_96.bin new file mode 100644 index 0000000000000000000000000000000000000000..6c1db5915ed946609748198525ed1afc7034fea2 --- /dev/null +++ b/params_shard_96.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3ac1372b1b4c39a352b4d7e9c913457217333d17e674d19e9004f94f759579c +size 50331648 diff --git a/params_shard_97.bin b/params_shard_97.bin new file mode 100644 index 0000000000000000000000000000000000000000..463b6fc93092f99d92eb3fd1807383e5ad66fb7c --- /dev/null +++ b/params_shard_97.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9541759d44749974b8a1585488928e23b82ab694d00ecfa38f40b4407d4df98 +size 25165824 diff --git a/params_shard_98.bin b/params_shard_98.bin new file mode 100644 index 0000000000000000000000000000000000000000..9d31a35732ba60da169c3d64fbf9cf30af69d3d6 --- /dev/null +++ b/params_shard_98.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcd728ed88d37bdb2ce8f3cfa536fc9719062028c2f164e105e6396d9fefd5c2 +size 50331648 diff --git a/params_shard_99.bin b/params_shard_99.bin new file mode 100644 index 0000000000000000000000000000000000000000..0078df32ff34e3721ec9cab94f8561bbf33a1112 --- /dev/null +++ b/params_shard_99.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbca303b5a42661611451fc8248f0244dc067ad6f9cc3705713d45b5759bb930 +size 31473920 diff --git a/tensor-cache-b16.json b/tensor-cache-b16.json new file mode 100644 index 0000000000000000000000000000000000000000..85f5bf08cfaf42a98eaac7d87a8d980c52fc75bd --- /dev/null +++ b/tensor-cache-b16.json @@ -0,0 +1,7263 @@ +{ + "metadata": { + "ParamSize": 581, + "ParamBytes": 5038043136.0, + "BitsPerParam": 4.283224533928671 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 508559360, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 248320, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 508559360, + "byteOffset": 0 + } + ], + "md5sum": "a94f9b5980767a55d5fd7b2b17c431bb" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 63569920, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 248320, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 63569920, + "byteOffset": 0 + } + ], + "md5sum": "84827c3652b4672fbbbdb8298a82b08f" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 508559360, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 248320, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 508559360, + "byteOffset": 0 + } + ], + "md5sum": "0b45ed0e64414adf0ab6d0a9ebed9387" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 63569920, + "records": [ + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 248320, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 63569920, + "byteOffset": 0 + } + ], + "md5sum": "78df488bd25a8c5e3741a2fe13042d79" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 28533120, + "records": [ + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.0.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 8192 + }, + { + "name": "model.layers.0.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 8256 + }, + { + "name": "model.layers.0.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 73792 + }, + { + "name": "model.layers.0.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 73856 + }, + { + "name": "model.layers.0.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 139392 + }, + { + "name": "model.layers.0.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 147584 + }, + { + "name": "model.layers.0.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 213120 + }, + { + "name": "model.layers.0.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 221312 + }, + { + "name": "model.layers.0.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16998528 + }, + { + "name": "model.layers.0.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 19095680 + }, + { + "name": "model.layers.0.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27484288 + }, + { + "name": "model.layers.0.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 28532864 + } + ], + "md5sum": "0c8585ccc13df697bc19fe3809ede736" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2aebbbe20c98710aea73e186d61c9739" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "6f7279f270a2030ee74b3e2fd413a13b" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.1.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "2d194413bb09449ababd3a3d0428c4db" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.0.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.0.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 9437184 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 12582912 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18874368 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18882560 + }, + { + "name": "model.layers.1.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 18890752 + }, + { + "name": "model.layers.1.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18890816 + }, + { + "name": "model.layers.1.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 18956352 + }, + { + "name": "model.layers.1.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18956416 + }, + { + "name": "model.layers.1.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19021952 + }, + { + "name": "model.layers.1.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 19030144 + }, + { + "name": "model.layers.1.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19095680 + }, + { + "name": "model.layers.1.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 19103872 + }, + { + "name": "model.layers.1.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 21201024 + }, + { + "name": "model.layers.1.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 29589632 + }, + { + "name": "model.layers.1.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 30638208 + } + ], + "md5sum": "a41606b0b15f2a268ae47e8bb4adc824" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "677e6fa07aa9e63da87162d381db7cd0" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d6acd9037df3603df52391e23f19af57" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.10.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "381859a917e45ff2d021f7e2dbcee317" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.1.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.1.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 9437184 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 12582912 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18874368 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18882560 + }, + { + "name": "model.layers.10.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 18890752 + }, + { + "name": "model.layers.10.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18890816 + }, + { + "name": "model.layers.10.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 18956352 + }, + { + "name": "model.layers.10.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18956416 + }, + { + "name": "model.layers.10.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19021952 + }, + { + "name": "model.layers.10.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 19030144 + }, + { + "name": "model.layers.10.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19095680 + }, + { + "name": "model.layers.10.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 19103872 + }, + { + "name": "model.layers.10.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 21201024 + }, + { + "name": "model.layers.10.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 29589632 + }, + { + "name": "model.layers.10.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 30638208 + } + ], + "md5sum": "cfe5c36fe0632dea6326e2b4ec2a7bc8" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "16e648224811a5e9ec3fe0b2ebb63b35" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "736120c63d879f2f418bfa9ae0e7ab5f" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2746d200cc96d1b9cfd7a3fbd7a5837a" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "e6fb6b16f44c3c7370b64afeb63d214e" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 20971520, + "records": [ + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 0 + } + ], + "md5sum": "b81636ef00ad3c1503bf44babb340750" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 30958080, + "records": [ + { + "name": "model.layers.10.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.10.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 9437184 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 12582912 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18874368 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18882560 + }, + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18890752 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22036480 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28327936 + }, + { + "name": "model.layers.11.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 28336128 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 28336640 + } + ], + "md5sum": "bcaa98dca1ed139fc804e497fce192f9" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.11.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.12.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.12.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.12.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.12.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.12.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.12.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.12.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.12.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.12.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "dec4e5543e2f7c80d62fb1f6c7db7024" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "29920a245fe14df2558d36eab1a3601c" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "1601445307f4f68c1079924761cdb4a1" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.13.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "2564ff1d34566107920b086b9c259b04" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.12.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.12.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.12.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.12.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.12.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.13.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.13.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.13.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.13.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.13.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.13.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.13.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.13.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "ffad61177cf2a98bb7282bc7b7e72b37" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "6f89d71e01ba2876c7fe506cbd1314fc" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "b670dcb3eb54bac129eceb7cd13112d5" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.14.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "b09fbb84bf9f1d6fadc2c297ada27286" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.13.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.13.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.13.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.13.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.13.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.14.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.14.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.14.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.14.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.14.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.14.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.14.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.14.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "94d061e282632f7427bd53cf8a8c3c11" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c2fd46ae51094ed3d3e4f2b9927d5ba7" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "f9c0be36d22d6742e9e62db5c20eddae" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "b2e03d11f3696daafd6a5ec35d1db094" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "b63d7efbc0fa85a23ab840579b87db81" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.14.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.14.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.14.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.14.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.14.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "c18b577cfe8598741ba79e84597b8c52" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.15.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "0387632320a135969574d4fd5d0f500d" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.15.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.16.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.16.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.16.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.16.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.16.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.16.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.16.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.16.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.16.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "5f840ffe1281d3feb0e4c2a70ad0dc55" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "8bff43ec5e944a1fe584ebe99e359abd" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "36a43d0e26da90318f978e59d51a50c2" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.17.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "175ea2a56482277a994305ce40af2f2c" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.16.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.16.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.16.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.16.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.16.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.17.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.17.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.17.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.17.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.17.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.17.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.17.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.17.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "4899832ed295f230a8c5f946eba70058" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "eb4e9d4dafe0a2df254461425dcd7d12" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "8bd1dd35622c5796ea93cea967d367be" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.18.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "b0e74f730499e3c5b15f7724b415deb7" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.17.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.17.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.17.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.17.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.17.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.18.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.18.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.18.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.18.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.18.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.18.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.18.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.18.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "c0605649a30d61d6006bbea584b175b7" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "053b04afbf8f8eceaefcc10eaa45802f" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "5a4f7bf549a6d165cedca58f9e22b589" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "287c6c48e4286aa5458eb2a6d8a7a1f0" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "10a19d621a751544ce045f09df58329c" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.18.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.18.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.18.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.18.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.18.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "08827220d4357aca50e4249b8e424387" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.19.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "b5868a8c370e463ad01de5d23c04c8a3" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.19.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.2.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.2.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.2.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.2.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.2.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.2.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.2.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.2.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.2.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "9160d3e755bbc6791f952291ab3427dc" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "519c9426a5f0dc77e9df71fac810aeb5" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "82b40b1a84f175bc7e51e8eba0e22a3b" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.20.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "567f03a248ecb218bd2b6d2fe924307a" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.2.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.2.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.2.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.2.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.2.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.20.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.20.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.20.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.20.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.20.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.20.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.20.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.20.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "abb57be32dca3c73290c0d9be10774b1" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "50e4112a298b6aa21eaf36fc1ed2e764" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "00838603b7d353d6092fe25fc5ddb418" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.21.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "fdbf9c88c32c7f433457c09b1150ee2c" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.20.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.20.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.20.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.20.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.20.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.21.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.21.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.21.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.21.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.21.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.21.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.21.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.21.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "7fb09a7981c567fa1ce1ae1938ed9b25" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "920af0fcdefc2272bf666d96fc51b1c9" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "41163a03e7fc49bb1180ede30ae6e009" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.22.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "054ec483064a0b29cfb15f4270f68241" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.21.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.21.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.21.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.21.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.21.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.22.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.22.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.22.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.22.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.22.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.22.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.22.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.22.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "d67bdd362a1b393759c557447d14c095" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "3899779895e8c9ad4f7b0a3770383bea" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "928594805c0f92093ed41ba047c9471f" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d763907edb31ba123b5b45806c76436d" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "947043481286800308b3a4bad60b0c57" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.22.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.22.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.22.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.22.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.22.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "089151c7208f29ab4740af3c5fb24fc5" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.23.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "1dc90a48726c9a7d4045c27e5ebc0c84" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.23.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.24.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.24.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.24.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.24.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.24.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.24.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.24.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.24.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.24.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "9531d1dbde851abf65f3323ecd3d4e45" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2478354bfe9eeca369a9358ca6b3e854" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "bcca03fa5957d3248356109c34e77f79" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.25.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "71ef4169f74cb691c62426ccca0f035a" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.24.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.24.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.24.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.24.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.24.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.25.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.25.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.25.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.25.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.25.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.25.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.25.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.25.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "d49eafd262200b3ecf4d590f5a21d992" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1fcc359af83bd7b81a0ec4f010048c51" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d307b03ee8716058a3ce61f7d77db7b7" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.26.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "524681cb7f724b885b09556047d3cc4d" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.25.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.25.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.25.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.25.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.25.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.26.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.26.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.26.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.26.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.26.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.26.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.26.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.26.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "8e16c5166b8aa87b3a002757d2c79e90" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e65e21fd7c89002dd204d7acc437b7fc" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "55c38ee1f6425e45e4d91e97da433e1e" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f6e2620c33f625c8562e50f57b23b599" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "89bc00764c7a6610b274d195b9d90ec5" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.26.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.26.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.26.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.26.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.26.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "34922020fcb55c9fd80ce06dd5efa1ab" + }, + { + "dataPath": "params_shard_82.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.27.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "1359e5fcbee81f028b23ee63b31d24de" + }, + { + "dataPath": "params_shard_83.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.27.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.28.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.28.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.28.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.28.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.28.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.28.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.28.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.28.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.28.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.28.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "8205bacab431a82bf4c32123eb42914e" + }, + { + "dataPath": "params_shard_84.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.28.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2f5588d152df4de818c020b7cc74327c" + }, + { + "dataPath": "params_shard_85.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.28.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "c08fe0c0b6d61ddb51e69e2f374d4129" + }, + { + "dataPath": "params_shard_86.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.29.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "0f20364321834ebc2addc1665019c564" + }, + { + "dataPath": "params_shard_87.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.28.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.28.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.28.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.28.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.28.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.28.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.28.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.28.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.29.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.29.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.29.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.29.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.29.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.29.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.29.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.29.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.29.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "daa73f71e9bcb582351cfcb125184308" + }, + { + "dataPath": "params_shard_88.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.29.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "3b29c3f218a9e053c063b03068bf074f" + }, + { + "dataPath": "params_shard_89.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.29.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "ea3223fd2b7e911436e477fe91a35aa9" + }, + { + "dataPath": "params_shard_90.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "8be76d1fc4743107c7be0b30cfe0a100" + }, + { + "dataPath": "params_shard_91.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d422495eea05cafd111985b05890c609" + }, + { + "dataPath": "params_shard_92.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.29.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.29.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.29.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.29.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.29.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.29.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.29.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.29.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "54ce1f6c98f1e014ac339492900adbf5" + }, + { + "dataPath": "params_shard_93.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.3.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "b3b659455e24748903d6663813c25aab" + }, + { + "dataPath": "params_shard_94.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.3.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.30.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.30.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.30.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.30.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.30.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.30.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.30.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.30.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.30.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.30.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "e819b22cc7c4614b245ff6d42de1a493" + }, + { + "dataPath": "params_shard_95.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.30.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "ae37096d5f826dd59233ec665b4b877b" + }, + { + "dataPath": "params_shard_96.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.30.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "b0a65386dc14995aca6a4079c895ae9d" + }, + { + "dataPath": "params_shard_97.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.31.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e7cb3c326458f6bff8721b999f5334b3" + }, + { + "dataPath": "params_shard_98.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.31.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "9643c201e88d66b7f45ec8ee405adf20" + }, + { + "dataPath": "params_shard_99.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.30.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.30.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.30.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.30.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.30.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.30.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.30.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.30.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.31.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.31.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "a7142893e4a50dc65ac1a1b879554700" + }, + { + "dataPath": "params_shard_100.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.31.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.31.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.31.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.31.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.31.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "81315694d372c7d034fdc3174c4a5674" + }, + { + "dataPath": "params_shard_101.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.31.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.31.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.31.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.4.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.4.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.4.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.4.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.4.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.4.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.4.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.4.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.4.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "921bda7e5886a19918ce2a0fbeafb501" + }, + { + "dataPath": "params_shard_102.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "74d1f22849f60bc05028611de1eda841" + }, + { + "dataPath": "params_shard_103.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "773e47e1a74fda7601561b3be19ac1d0" + }, + { + "dataPath": "params_shard_104.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.5.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "37e49ba64d09c3d461b5f3316326a5d3" + }, + { + "dataPath": "params_shard_105.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.4.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.4.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.4.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.4.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.4.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.5.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.5.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.5.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.5.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.5.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.5.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.5.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.5.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "82adecafcadaaebb96c216bd504d6d63" + }, + { + "dataPath": "params_shard_106.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1be9bbde503f9dc984d0a63b403c801c" + }, + { + "dataPath": "params_shard_107.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "04021f25ed4f9baff0df4821b61e5052" + }, + { + "dataPath": "params_shard_108.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.6.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "80858669e6ad1b6b2159653da7c75e31" + }, + { + "dataPath": "params_shard_109.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.5.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.5.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.5.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.5.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.5.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.6.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.6.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.6.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.6.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.6.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.6.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.6.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.6.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "a45ad6aea4f366efd4fa123d26722760" + }, + { + "dataPath": "params_shard_110.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "9a9097ea4d12da2e3e3ac0a35ad56c3c" + }, + { + "dataPath": "params_shard_111.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "bb00c7bd755d64401a6890c69bffda1d" + }, + { + "dataPath": "params_shard_112.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "99bed590cf2d06bdef0ced7c5aad3f56" + }, + { + "dataPath": "params_shard_113.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "90c668f1c81c7d19a675c2f0429d13fa" + }, + { + "dataPath": "params_shard_114.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.6.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.6.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.6.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.6.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.6.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "ff4dfd139cbcd65c75321cb53a803b57" + }, + { + "dataPath": "params_shard_115.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.7.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "bf2f0eea75a062436bb1d128ab5e015e" + }, + { + "dataPath": "params_shard_116.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.7.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.8.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.8.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.8.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.8.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.8.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.8.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.8.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.8.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.8.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "3a62d8dea42694d88201fcdc9b2d6bb9" + }, + { + "dataPath": "params_shard_117.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "14b756076df54c35a5e2ccff6a125ce1" + }, + { + "dataPath": "params_shard_118.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d6e78185f2f28f323fd3e4d554c8ca8b" + }, + { + "dataPath": "params_shard_119.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.9.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "e86f2db51741a83b1b37ad8f6a02d5aa" + }, + { + "dataPath": "params_shard_120.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.8.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.8.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.8.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.8.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.8.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.9.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.9.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.9.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "bfloat16", + "format": "raw", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.9.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.9.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.9.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.9.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.9.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "2f07d611ee94ad53c576ae04de173bb7" + }, + { + "dataPath": "params_shard_121.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "99a9f35bc74057104cbc534836c961f7" + }, + { + "dataPath": "params_shard_122.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "4217ab9960b1db2807e420beddd6db4e" + }, + { + "dataPath": "params_shard_123.bin", + "format": "raw-shard", + "nbytes": 28328192, + "records": [ + { + "name": "model.layers.9.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.9.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.9.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.9.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.9.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.norm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + } + ], + "md5sum": "22d57669255f85566c6873459ac816b9" + } + ] +} \ No newline at end of file diff --git a/tensor-cache.json b/tensor-cache.json new file mode 100644 index 0000000000000000000000000000000000000000..34d4fee51952edc50dc8486dc8d72a335c2d7750 --- /dev/null +++ b/tensor-cache.json @@ -0,0 +1,7263 @@ +{ + "metadata": { + "ParamSize": 581, + "ParamBytes": 5038043136.0, + "BitsPerParam": 4.283224533928671 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 508559360, + "records": [ + { + "name": "lm_head.q_weight", + "shape": [ + 248320, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 508559360, + "byteOffset": 0 + } + ], + "md5sum": "a94f9b5980767a55d5fd7b2b17c431bb" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 63569920, + "records": [ + { + "name": "lm_head.q_scale", + "shape": [ + 248320, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 63569920, + "byteOffset": 0 + } + ], + "md5sum": "84827c3652b4672fbbbdb8298a82b08f" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 508559360, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 248320, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 508559360, + "byteOffset": 0 + } + ], + "md5sum": "0b45ed0e64414adf0ab6d0a9ebed9387" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 63569920, + "records": [ + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 248320, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 63569920, + "byteOffset": 0 + } + ], + "md5sum": "78df488bd25a8c5e3741a2fe13042d79" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 28533120, + "records": [ + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 0 + }, + { + "name": "model.layers.0.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 8192 + }, + { + "name": "model.layers.0.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 8256 + }, + { + "name": "model.layers.0.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 73792 + }, + { + "name": "model.layers.0.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 73856 + }, + { + "name": "model.layers.0.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 139392 + }, + { + "name": "model.layers.0.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 147584 + }, + { + "name": "model.layers.0.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 213120 + }, + { + "name": "model.layers.0.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 221312 + }, + { + "name": "model.layers.0.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 16998528 + }, + { + "name": "model.layers.0.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 19095680 + }, + { + "name": "model.layers.0.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 27484288 + }, + { + "name": "model.layers.0.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 28532864 + } + ], + "md5sum": "0c8585ccc13df697bc19fe3809ede736" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2aebbbe20c98710aea73e186d61c9739" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "6f7279f270a2030ee74b3e2fd413a13b" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.1.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "2d194413bb09449ababd3a3d0428c4db" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.0.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.0.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 9437184 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 12582912 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18874368 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18882560 + }, + { + "name": "model.layers.1.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 18890752 + }, + { + "name": "model.layers.1.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18890816 + }, + { + "name": "model.layers.1.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 18956352 + }, + { + "name": "model.layers.1.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18956416 + }, + { + "name": "model.layers.1.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19021952 + }, + { + "name": "model.layers.1.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 19030144 + }, + { + "name": "model.layers.1.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19095680 + }, + { + "name": "model.layers.1.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 19103872 + }, + { + "name": "model.layers.1.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 21201024 + }, + { + "name": "model.layers.1.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 29589632 + }, + { + "name": "model.layers.1.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 30638208 + } + ], + "md5sum": "a41606b0b15f2a268ae47e8bb4adc824" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "677e6fa07aa9e63da87162d381db7cd0" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d6acd9037df3603df52391e23f19af57" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.10.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "381859a917e45ff2d021f7e2dbcee317" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.1.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.1.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 9437184 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 12582912 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18874368 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18882560 + }, + { + "name": "model.layers.10.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 18890752 + }, + { + "name": "model.layers.10.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18890816 + }, + { + "name": "model.layers.10.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 18956352 + }, + { + "name": "model.layers.10.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 18956416 + }, + { + "name": "model.layers.10.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19021952 + }, + { + "name": "model.layers.10.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 19030144 + }, + { + "name": "model.layers.10.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 19095680 + }, + { + "name": "model.layers.10.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 19103872 + }, + { + "name": "model.layers.10.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 21201024 + }, + { + "name": "model.layers.10.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 29589632 + }, + { + "name": "model.layers.10.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 30638208 + } + ], + "md5sum": "cfe5c36fe0632dea6326e2b4ec2a7bc8" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "16e648224811a5e9ec3fe0b2ebb63b35" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "736120c63d879f2f418bfa9ae0e7ab5f" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2746d200cc96d1b9cfd7a3fbd7a5837a" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "e6fb6b16f44c3c7370b64afeb63d214e" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 20971520, + "records": [ + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 0 + } + ], + "md5sum": "b81636ef00ad3c1503bf44babb340750" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 30958080, + "records": [ + { + "name": "model.layers.10.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.10.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 9437184 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 12582912 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18874368 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 18882560 + }, + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18890752 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22036480 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28327936 + }, + { + "name": "model.layers.11.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 28336128 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 28336640 + } + ], + "md5sum": "bcaa98dca1ed139fc804e497fce192f9" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.11.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.12.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.12.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.12.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.12.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.12.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.12.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.12.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.12.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.12.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "dec4e5543e2f7c80d62fb1f6c7db7024" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "29920a245fe14df2558d36eab1a3601c" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "1601445307f4f68c1079924761cdb4a1" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.13.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "2564ff1d34566107920b086b9c259b04" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.12.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.12.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.12.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.12.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.12.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.13.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.13.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.13.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.13.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.13.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.13.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.13.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.13.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "ffad61177cf2a98bb7282bc7b7e72b37" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "6f89d71e01ba2876c7fe506cbd1314fc" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "b670dcb3eb54bac129eceb7cd13112d5" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.14.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "b09fbb84bf9f1d6fadc2c297ada27286" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.13.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.13.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.13.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.13.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.13.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.14.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.14.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.14.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.14.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.14.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.14.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.14.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.14.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "94d061e282632f7427bd53cf8a8c3c11" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "c2fd46ae51094ed3d3e4f2b9927d5ba7" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "f9c0be36d22d6742e9e62db5c20eddae" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "b2e03d11f3696daafd6a5ec35d1db094" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "b63d7efbc0fa85a23ab840579b87db81" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.14.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.14.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.14.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.14.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.14.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "c18b577cfe8598741ba79e84597b8c52" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.15.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "0387632320a135969574d4fd5d0f500d" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.15.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.16.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.16.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.16.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.16.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.16.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.16.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.16.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.16.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.16.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "5f840ffe1281d3feb0e4c2a70ad0dc55" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "8bff43ec5e944a1fe584ebe99e359abd" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "36a43d0e26da90318f978e59d51a50c2" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.17.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "175ea2a56482277a994305ce40af2f2c" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.16.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.16.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.16.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.16.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.16.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.17.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.17.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.17.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.17.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.17.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.17.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.17.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.17.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "4899832ed295f230a8c5f946eba70058" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "eb4e9d4dafe0a2df254461425dcd7d12" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "8bd1dd35622c5796ea93cea967d367be" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.18.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "b0e74f730499e3c5b15f7724b415deb7" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.17.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.17.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.17.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.17.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.17.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.18.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.18.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.18.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.18.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.18.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.18.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.18.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.18.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "c0605649a30d61d6006bbea584b175b7" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "053b04afbf8f8eceaefcc10eaa45802f" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "5a4f7bf549a6d165cedca58f9e22b589" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "287c6c48e4286aa5458eb2a6d8a7a1f0" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "10a19d621a751544ce045f09df58329c" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.18.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.18.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.18.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.18.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.18.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "08827220d4357aca50e4249b8e424387" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.19.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "b5868a8c370e463ad01de5d23c04c8a3" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.19.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.2.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.2.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.2.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.2.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.2.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.2.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.2.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.2.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.2.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "9160d3e755bbc6791f952291ab3427dc" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "519c9426a5f0dc77e9df71fac810aeb5" + }, + { + "dataPath": "params_shard_51.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "82b40b1a84f175bc7e51e8eba0e22a3b" + }, + { + "dataPath": "params_shard_52.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.20.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "567f03a248ecb218bd2b6d2fe924307a" + }, + { + "dataPath": "params_shard_53.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.2.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.2.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.2.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.2.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.2.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.20.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.20.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.20.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.20.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.20.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.20.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.20.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.20.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "abb57be32dca3c73290c0d9be10774b1" + }, + { + "dataPath": "params_shard_54.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "50e4112a298b6aa21eaf36fc1ed2e764" + }, + { + "dataPath": "params_shard_55.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "00838603b7d353d6092fe25fc5ddb418" + }, + { + "dataPath": "params_shard_56.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.21.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "fdbf9c88c32c7f433457c09b1150ee2c" + }, + { + "dataPath": "params_shard_57.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.20.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.20.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.20.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.20.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.20.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.21.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.21.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.21.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.21.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.21.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.21.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.21.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.21.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "7fb09a7981c567fa1ce1ae1938ed9b25" + }, + { + "dataPath": "params_shard_58.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "920af0fcdefc2272bf666d96fc51b1c9" + }, + { + "dataPath": "params_shard_59.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "41163a03e7fc49bb1180ede30ae6e009" + }, + { + "dataPath": "params_shard_60.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.22.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "054ec483064a0b29cfb15f4270f68241" + }, + { + "dataPath": "params_shard_61.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.21.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.21.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.21.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.21.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.21.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.22.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.22.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.22.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.22.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.22.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.22.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.22.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.22.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "d67bdd362a1b393759c557447d14c095" + }, + { + "dataPath": "params_shard_62.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "3899779895e8c9ad4f7b0a3770383bea" + }, + { + "dataPath": "params_shard_63.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "928594805c0f92093ed41ba047c9471f" + }, + { + "dataPath": "params_shard_64.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "d763907edb31ba123b5b45806c76436d" + }, + { + "dataPath": "params_shard_65.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "947043481286800308b3a4bad60b0c57" + }, + { + "dataPath": "params_shard_66.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.22.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.22.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.22.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.22.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.22.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "089151c7208f29ab4740af3c5fb24fc5" + }, + { + "dataPath": "params_shard_67.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.23.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "1dc90a48726c9a7d4045c27e5ebc0c84" + }, + { + "dataPath": "params_shard_68.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.23.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.24.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.24.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.24.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.24.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.24.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.24.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.24.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.24.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.24.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "9531d1dbde851abf65f3323ecd3d4e45" + }, + { + "dataPath": "params_shard_69.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2478354bfe9eeca369a9358ca6b3e854" + }, + { + "dataPath": "params_shard_70.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "bcca03fa5957d3248356109c34e77f79" + }, + { + "dataPath": "params_shard_71.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.25.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "71ef4169f74cb691c62426ccca0f035a" + }, + { + "dataPath": "params_shard_72.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.24.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.24.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.24.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.24.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.24.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.25.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.25.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.25.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.25.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.25.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.25.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.25.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.25.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "d49eafd262200b3ecf4d590f5a21d992" + }, + { + "dataPath": "params_shard_73.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1fcc359af83bd7b81a0ec4f010048c51" + }, + { + "dataPath": "params_shard_74.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d307b03ee8716058a3ce61f7d77db7b7" + }, + { + "dataPath": "params_shard_75.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.26.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "524681cb7f724b885b09556047d3cc4d" + }, + { + "dataPath": "params_shard_76.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.25.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.25.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.25.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.25.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.25.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.26.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.26.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.26.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.26.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.26.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.26.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.26.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.26.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "8e16c5166b8aa87b3a002757d2c79e90" + }, + { + "dataPath": "params_shard_77.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e65e21fd7c89002dd204d7acc437b7fc" + }, + { + "dataPath": "params_shard_78.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "55c38ee1f6425e45e4d91e97da433e1e" + }, + { + "dataPath": "params_shard_79.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "f6e2620c33f625c8562e50f57b23b599" + }, + { + "dataPath": "params_shard_80.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "89bc00764c7a6610b274d195b9d90ec5" + }, + { + "dataPath": "params_shard_81.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.26.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.26.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.26.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.26.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.26.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "34922020fcb55c9fd80ce06dd5efa1ab" + }, + { + "dataPath": "params_shard_82.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.27.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "1359e5fcbee81f028b23ee63b31d24de" + }, + { + "dataPath": "params_shard_83.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.27.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.28.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.28.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.28.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.28.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.28.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.28.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.28.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.28.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.28.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.28.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "8205bacab431a82bf4c32123eb42914e" + }, + { + "dataPath": "params_shard_84.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.28.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "2f5588d152df4de818c020b7cc74327c" + }, + { + "dataPath": "params_shard_85.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.28.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "c08fe0c0b6d61ddb51e69e2f374d4129" + }, + { + "dataPath": "params_shard_86.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.29.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "0f20364321834ebc2addc1665019c564" + }, + { + "dataPath": "params_shard_87.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.28.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.28.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.28.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.28.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.28.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.28.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.28.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.28.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.29.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.29.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.29.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.29.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.29.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.29.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.29.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.29.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.29.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "daa73f71e9bcb582351cfcb125184308" + }, + { + "dataPath": "params_shard_88.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.29.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "3b29c3f218a9e053c063b03068bf074f" + }, + { + "dataPath": "params_shard_89.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.29.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "ea3223fd2b7e911436e477fe91a35aa9" + }, + { + "dataPath": "params_shard_90.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "8be76d1fc4743107c7be0b30cfe0a100" + }, + { + "dataPath": "params_shard_91.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d422495eea05cafd111985b05890c609" + }, + { + "dataPath": "params_shard_92.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.29.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.29.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.29.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.29.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.29.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.29.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.29.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.29.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "54ce1f6c98f1e014ac339492900adbf5" + }, + { + "dataPath": "params_shard_93.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.3.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "b3b659455e24748903d6663813c25aab" + }, + { + "dataPath": "params_shard_94.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.3.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.30.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.30.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.30.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.30.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.30.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.30.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.30.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.30.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.30.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.30.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "e819b22cc7c4614b245ff6d42de1a493" + }, + { + "dataPath": "params_shard_95.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.30.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "ae37096d5f826dd59233ec665b4b877b" + }, + { + "dataPath": "params_shard_96.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.30.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "b0a65386dc14995aca6a4079c895ae9d" + }, + { + "dataPath": "params_shard_97.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.31.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "e7cb3c326458f6bff8721b999f5334b3" + }, + { + "dataPath": "params_shard_98.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.31.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "9643c201e88d66b7f45ec8ee405adf20" + }, + { + "dataPath": "params_shard_99.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.30.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.30.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.30.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.30.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.30.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.30.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.30.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.30.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.31.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.31.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "a7142893e4a50dc65ac1a1b879554700" + }, + { + "dataPath": "params_shard_100.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.31.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.31.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.31.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.31.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.31.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "81315694d372c7d034fdc3174c4a5674" + }, + { + "dataPath": "params_shard_101.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.31.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.31.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.31.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.4.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.4.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.4.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.4.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.4.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.4.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.4.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.4.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.4.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "921bda7e5886a19918ce2a0fbeafb501" + }, + { + "dataPath": "params_shard_102.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "74d1f22849f60bc05028611de1eda841" + }, + { + "dataPath": "params_shard_103.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "773e47e1a74fda7601561b3be19ac1d0" + }, + { + "dataPath": "params_shard_104.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.5.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "37e49ba64d09c3d461b5f3316326a5d3" + }, + { + "dataPath": "params_shard_105.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.4.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.4.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.4.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.4.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.4.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.5.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.5.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.5.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.5.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.5.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.5.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.5.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.5.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "82adecafcadaaebb96c216bd504d6d63" + }, + { + "dataPath": "params_shard_106.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "1be9bbde503f9dc984d0a63b403c801c" + }, + { + "dataPath": "params_shard_107.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "04021f25ed4f9baff0df4821b61e5052" + }, + { + "dataPath": "params_shard_108.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.6.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "80858669e6ad1b6b2159653da7c75e31" + }, + { + "dataPath": "params_shard_109.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.5.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.5.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.5.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.5.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.5.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.6.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.6.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.6.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.6.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.6.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.6.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.6.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.6.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "a45ad6aea4f366efd4fa123d26722760" + }, + { + "dataPath": "params_shard_110.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "9a9097ea4d12da2e3e3ac0a35ad56c3c" + }, + { + "dataPath": "params_shard_111.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "bb00c7bd755d64401a6890c69bffda1d" + }, + { + "dataPath": "params_shard_112.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "99bed590cf2d06bdef0ced7c5aad3f56" + }, + { + "dataPath": "params_shard_113.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "90c668f1c81c7d19a675c2f0429d13fa" + }, + { + "dataPath": "params_shard_114.bin", + "format": "raw-shard", + "nbytes": 31473920, + "records": [ + { + "name": "model.layers.6.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.6.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.6.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.6.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.6.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 28328192 + } + ], + "md5sum": "ff4dfd139cbcd65c75321cb53a803b57" + }, + { + "dataPath": "params_shard_115.bin", + "format": "raw-shard", + "nbytes": 29893120, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 0 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 6291456 + }, + { + "name": "model.layers.7.self_attn.k_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 6299648 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 10240, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 20971520, + "byteOffset": 6300160 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 10240, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2621440, + "byteOffset": 27271680 + } + ], + "md5sum": "bf2f0eea75a062436bb1d128ab5e015e" + }, + { + "dataPath": "params_shard_116.bin", + "format": "raw-shard", + "nbytes": 28533376, + "records": [ + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.7.self_attn.q_norm.weight", + "shape": [ + 256 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 512, + "byteOffset": 9437184 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9437696 + }, + { + "name": "model.layers.8.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9445888 + }, + { + "name": "model.layers.8.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9445952 + }, + { + "name": "model.layers.8.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 9511488 + }, + { + "name": "model.layers.8.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9511552 + }, + { + "name": "model.layers.8.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9577088 + }, + { + "name": "model.layers.8.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 9585280 + }, + { + "name": "model.layers.8.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 9650816 + }, + { + "name": "model.layers.8.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 9659008 + }, + { + "name": "model.layers.8.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 26436224 + } + ], + "md5sum": "3a62d8dea42694d88201fcdc9b2d6bb9" + }, + { + "dataPath": "params_shard_117.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "14b756076df54c35a5e2ccff6a125ce1" + }, + { + "dataPath": "params_shard_118.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "d6e78185f2f28f323fd3e4d554c8ca8b" + }, + { + "dataPath": "params_shard_119.bin", + "format": "raw-shard", + "nbytes": 16777216, + "records": [ + { + "name": "model.layers.9.linear_attn.in_proj_qkv.q_weight", + "shape": [ + 8192, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 16777216, + "byteOffset": 0 + } + ], + "md5sum": "e86f2db51741a83b1b37ad8f6a02d5aa" + }, + { + "dataPath": "params_shard_120.bin", + "format": "raw-shard", + "nbytes": 30638464, + "records": [ + { + "name": "model.layers.8.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.8.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.8.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.8.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.8.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + }, + { + "name": "model.layers.9.linear_attn.A_log", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28328192 + }, + { + "name": "model.layers.9.linear_attn.conv1d_weight", + "shape": [ + 8192, + 1, + 4 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28328256 + }, + { + "name": "model.layers.9.linear_attn.dt_bias", + "shape": [ + 32 + ], + "dtype": "float32", + "format": "f32-to-bf16", + "nbytes": 64, + "byteOffset": 28393792 + }, + { + "name": "model.layers.9.linear_attn.in_proj_a.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28393856 + }, + { + "name": "model.layers.9.linear_attn.in_proj_a.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28459392 + }, + { + "name": "model.layers.9.linear_attn.in_proj_b.q_weight", + "shape": [ + 32, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 65536, + "byteOffset": 28467584 + }, + { + "name": "model.layers.9.linear_attn.in_proj_b.q_scale", + "shape": [ + 32, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28533120 + }, + { + "name": "model.layers.9.linear_attn.in_proj_qkv.q_scale", + "shape": [ + 8192, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 2097152, + "byteOffset": 28541312 + } + ], + "md5sum": "2f07d611ee94ad53c576ae04de173bb7" + }, + { + "dataPath": "params_shard_121.bin", + "format": "raw-shard", + "nbytes": 25165824, + "records": [ + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 4096, + 1536 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 25165824, + "byteOffset": 0 + } + ], + "md5sum": "99a9f35bc74057104cbc534836c961f7" + }, + { + "dataPath": "params_shard_122.bin", + "format": "raw-shard", + "nbytes": 50331648, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 24576, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 50331648, + "byteOffset": 0 + } + ], + "md5sum": "4217ab9960b1db2807e420beddd6db4e" + }, + { + "dataPath": "params_shard_123.bin", + "format": "raw-shard", + "nbytes": 28328192, + "records": [ + { + "name": "model.layers.9.linear_attn.in_proj_z.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 0 + }, + { + "name": "model.layers.9.linear_attn.in_proj_z.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 8388608 + }, + { + "name": "model.layers.9.linear_attn.norm.weight", + "shape": [ + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 256, + "byteOffset": 9437184 + }, + { + "name": "model.layers.9.linear_attn.out_proj.q_weight", + "shape": [ + 4096, + 512 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 8388608, + "byteOffset": 9437440 + }, + { + "name": "model.layers.9.linear_attn.out_proj.q_scale", + "shape": [ + 4096, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1048576, + "byteOffset": 17826048 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 4096, + 384 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3145728, + "byteOffset": 18874624 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 24576, + 128 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 6291456, + "byteOffset": 22020352 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28311808 + }, + { + "name": "model.norm.weight", + "shape": [ + 4096 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 8192, + "byteOffset": 28320000 + } + ], + "md5sum": "22d57669255f85566c6873459ac816b9" + } + ] +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..67741b04f23bfdb46501f748ce27865ec82eccfb --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4 +size 19989343 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6be6ce1780cf43bd47577fbb76e74aee6db89f21 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +}