Upload folder using huggingface_hub

README.md

@@ -0,0 +1,33 @@
+---
+library_name: keras-hub
+pipeline_tag: text-generation
+---
+This is a [`Qwen3Moe` model](https://keras.io/api/keras_hub/models/qwen3_moe) uploaded using the KerasHub library and can be used with JAX, TensorFlow, and PyTorch backends.
+This model is related to a `CausalLM` task.
+
+Model config:
+* **name:** qwen3_moe_backbone
+* **trainable:** True
+* **dtype:** {'module': 'keras', 'class_name': 'DTypePolicy', 'config': {'name': 'bfloat16'}, 'registered_name': None}
+* **vocabulary_size:** 151936
+* **num_layers:** 94
+* **num_query_heads:** 64
+* **head_dim:** 128
+* **hidden_dim:** 4096
+* **intermediate_dim:** 12288
+* **moe_intermediate_dim:** 1536
+* **rope_max_wavelength:** 1000000.0
+* **num_key_value_heads:** 4
+* **rope_scaling_factor:** 1.0
+* **layer_norm_epsilon:** 1e-06
+* **dropout:** 0
+* **tie_word_embeddings:** False
+* **sliding_window_size:** None
+* **num_experts:** 128
+* **top_k:** 8
+* **norm_top_k_prob:** True
+* **decoder_sparse_step:** 1
+* **mlp_only_layers:** []
+* **router_aux_loss_coefficient:** 0.001
+
+This model card has been generated automatically and should be completed by the model author. See [Model Cards documentation](https://huggingface.co/docs/hub/model-cards) for more information.

config.json

@@ -0,0 +1,37 @@
+{
+    "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_backbone",
+    "class_name": "Qwen3MoeBackbone",
+    "config": {
+        "name": "qwen3_moe_backbone",
+        "trainable": true,
+        "dtype": {
+            "module": "keras",
+            "class_name": "DTypePolicy",
+            "config": {
+                "name": "bfloat16"
+            },
+            "registered_name": null
+        },
+        "vocabulary_size": 151936,
+        "num_layers": 94,
+        "num_query_heads": 64,
+        "head_dim": 128,
+        "hidden_dim": 4096,
+        "intermediate_dim": 12288,
+        "moe_intermediate_dim": 1536,
+        "rope_max_wavelength": 1000000.0,
+        "num_key_value_heads": 4,
+        "rope_scaling_factor": 1.0,
+        "layer_norm_epsilon": 1e-06,
+        "dropout": 0,
+        "tie_word_embeddings": false,
+        "sliding_window_size": null,
+        "num_experts": 128,
+        "top_k": 8,
+        "norm_top_k_prob": true,
+        "decoder_sparse_step": 1,
+        "mlp_only_layers": [],
+        "router_aux_loss_coefficient": 0.001
+    },
+    "registered_name": "keras_hub>Qwen3MoeBackbone"
+}

metadata.json

@@ -0,0 +1,9 @@
+{
+    "keras_version": "3.11.3",
+    "keras_hub_version": "0.23.0.dev0",
+    "parameter_count": 235093634560,
+    "date_saved": "2025-10-07@19:39:37",
+    "tasks": [
+        "CausalLM"
+    ]
+}

model_00000.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bdaeb3b608a5b6c341c3a3bd43d0c3c9d11e6dc6fbf69679b7957dc3907c03a
+size 15661661840

model_00001.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4af259ee5a5e10864e6b8548725b6fd2772d94be971168e4677914da4017b43e
+size 14926655440

model_00002.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2d2c2b9062cd9f88990d7ea723a91b85849f3a2efe484e2542138959e0ea3c0
+size 14926655440

model_00003.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70a4721a6a9b36a4e4ece65009aaec72ba498cf97e7fcf43c9761eeab807e1f7
+size 14926655440

model_00004.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91de0ac1095ad69de2e118c1864dbe207d3f09c6a8f7ab2366a3b50362c14464
+size 14926655632

model_00005.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1493d873dcb343ab14f6d58c7d8c919f0b32c18bf6fae80243a2fe9161d72a57
+size 14926655632

model_00006.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d02093cde47b388375f09fadcb2c8c718d4269b4c7eca2beddf80f0dd85adb9a
+size 14926655632

model_00007.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3dafccaa6e49ca82de4c44f0bba468c6dfb6d1f2c79039924bac80fcbd4f0f1
+size 14926655632

model_00008.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ce10e5219dfe8c291474add9a4bad2e75f3efb6e33054a94a2a7ab72dcda43c
+size 14926655632

model_00009.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6482129e5764aec32234a327a794204ba13440912e22cffddb9e29710099da89
+size 14926655632

model_00010.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5566033e9c92ef0834558d7c9d25344914729b507f538de5cd528ec6b149c18a
+size 14926655632

model_00011.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eb4339025664291877a1c6eccad36dd940e6a5ac142b01b45f4e0dfc304950f
+size 14926655632

model_00012.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:548ae0c5fbe1bbcaf543ee6fcbb854f88eb80bf801668af737d2f14903fabe40
+size 14926655632

model_00013.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aad57dd7382d5683ccd6d12dac6257d9f7acfe07ab8ec3c4799d4b51c00ccf6d
+size 14926655632

model_00014.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72459f56f8d37a32ce9321ed7157d9f74f0cf8541e085cbbe2a9539b755cefa6
+size 14926655632

model_00015.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9569b8b5c76fc8c8cda211fdaa5cd72b6ce84d6f0931ff23896a58c3d5b9cf5e
+size 14926655632

model_00016.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bac9b4fdae119a12bd3b930268c6a4d7c9d2bc79ecb54fc2d34a07ae8fa704d
+size 14926655632

model_00017.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:397f9957d7031bad6a1483597e1d3121bfd12fdb5b5a9f652a8bd963c063b888
+size 14926655632

model_00018.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20db29d8470f890cfb81e3442caa521e3a2789f99897b95a1224c30c67054358
+size 14926655632

model_00019.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae1efc486bccc14be04d2faa7daf07693ae16e8138967607b80f26b881709305
+size 14926655632

model_00020.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:368228482d93ab3a0fc170cf87e995e4881095dd3c3fb80155fa7e5678862121
+size 14926655632

model_00021.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:702e84aa67bfa1c2fc67c23d3ac7b80e54b775d488c64771d1a5f290944fd9f7
+size 14926655632

model_00022.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a59efa422de21640373547cedcdc47410bcd8cb73748f4bca44d25695502b87
+size 14926655632

model_00023.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5cc784c9df80327ed6cf7f46a678f9e4d9ffb402b34c4f2c143d497bd93fa96
+size 14926655632

model_00024.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76a00a77144006f6a2372a6024498969e21a4dbb2c36d8254b25b217970e9501
+size 14926655632

model_00025.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46920d80b31edcfdafe1f69ec22ebf9a03ca67f5d9514b29389a9e0d9e037b4
+size 14926655632

model_00026.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5118c5c3765a6c90b6c5a1300935f66c8ff61186e7fe839e87118eb68481ec9d
+size 14926655632

model_00027.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0f0aaa7b6294d899e41d08c0a1cae3f0e3bf651bf655b2431094a39db14b966
+size 14926655632

model_00028.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de00734da638f303bbf73a99d5e279fe29ec98c7f10857b759436c4bd2102fa0
+size 14926655632

model_00029.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5be5b36711efad4c5b25ffd8343a6baff6a5c64f95ecbe1a0daa2565dc6e5e2
+size 14926655632

model_00030.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16a02528b26f9d2493eb04ffacfbeb7685ff5bd57af0e7ccc015642a1d242068
+size 14926655632

model_00031.weights.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d68c96e63daf05fdaacbc707b6b3eee24d3e184f4802015a9a54a80f7dbfe940
+size 6729886896

preprocessor.json

@@ -0,0 +1,69 @@
+{
+    "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_causal_lm_preprocessor",
+    "class_name": "Qwen3MoeCausalLMPreprocessor",
+    "config": {
+        "name": "qwen3_moe_causal_lm_preprocessor_2",
+        "trainable": true,
+        "dtype": {
+            "module": "keras",
+            "class_name": "DTypePolicy",
+            "config": {
+                "name": "float32"
+            },
+            "registered_name": null
+        },
+        "tokenizer": {
+            "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_tokenizer",
+            "class_name": "Qwen3MoeTokenizer",
+            "config": {
+                "name": "qwen3_moe_tokenizer",
+                "trainable": true,
+                "dtype": {
+                    "module": "keras",
+                    "class_name": "DTypePolicy",
+                    "config": {
+                        "name": "int32"
+                    },
+                    "registered_name": null
+                },
+                "config_file": "tokenizer.json",
+                "sequence_length": null,
+                "add_prefix_space": false,
+                "unsplittable_tokens": [
+                    "</tool_call>",
+                    "<|endoftext|>",
+                    "<|video_pad|>",
+                    "<think>",
+                    "<|im_end|>",
+                    "</think>",
+                    "<|file_sep|>",
+                    "<tool_call>",
+                    "<|object_ref_end|>",
+                    "<|fim_suffix|>",
+                    "<|quad_start|>",
+                    "<|image_pad|>",
+                    "<|object_ref_start|>",
+                    "<|im_start|>",
+                    "<|box_start|>",
+                    "<|fim_middle|>",
+                    "<|vision_end|>",
+                    "<|fim_pad|>",
+                    "<|box_end|>",
+                    "</tool_response>",
+                    "<|quad_end|>",
+                    "<|vision_pad|>",
+                    "<|fim_prefix|>",
+                    "<|vision_start|>",
+                    "<tool_response>",
+                    "<|repo_name|>"
+                ]
+            },
+            "registered_name": "keras_hub>Qwen3MoeTokenizer"
+        },
+        "config_file": "preprocessor.json",
+        "sequence_length": 1024,
+        "add_start_token": true,
+        "add_end_token": true
+    },
+    "registered_name": "keras_hub>Qwen3MoeCausalLMPreprocessor"
+}

task.json

@@ -0,0 +1,114 @@
+{
+    "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_causal_lm",
+    "class_name": "Qwen3MoeCausalLM",
+    "config": {
+        "backbone": {
+            "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_backbone",
+            "class_name": "Qwen3MoeBackbone",
+            "config": {
+                "name": "qwen3_moe_backbone",
+                "trainable": true,
+                "dtype": {
+                    "module": "keras",
+                    "class_name": "DTypePolicy",
+                    "config": {
+                        "name": "bfloat16"
+                    },
+                    "registered_name": null
+                },
+                "vocabulary_size": 151936,
+                "num_layers": 94,
+                "num_query_heads": 64,
+                "head_dim": 128,
+                "hidden_dim": 4096,
+                "intermediate_dim": 12288,
+                "moe_intermediate_dim": 1536,
+                "rope_max_wavelength": 1000000.0,
+                "num_key_value_heads": 4,
+                "rope_scaling_factor": 1.0,
+                "layer_norm_epsilon": 1e-06,
+                "dropout": 0,
+                "tie_word_embeddings": false,
+                "sliding_window_size": null,
+                "num_experts": 128,
+                "top_k": 8,
+                "norm_top_k_prob": true,
+                "decoder_sparse_step": 1,
+                "mlp_only_layers": [],
+                "router_aux_loss_coefficient": 0.001
+            },
+            "registered_name": "keras_hub>Qwen3MoeBackbone"
+        },
+        "preprocessor": {
+            "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_causal_lm_preprocessor",
+            "class_name": "Qwen3MoeCausalLMPreprocessor",
+            "config": {
+                "name": "qwen3_moe_causal_lm_preprocessor_2",
+                "trainable": true,
+                "dtype": {
+                    "module": "keras",
+                    "class_name": "DTypePolicy",
+                    "config": {
+                        "name": "float32"
+                    },
+                    "registered_name": null
+                },
+                "tokenizer": {
+                    "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_tokenizer",
+                    "class_name": "Qwen3MoeTokenizer",
+                    "config": {
+                        "name": "qwen3_moe_tokenizer",
+                        "trainable": true,
+                        "dtype": {
+                            "module": "keras",
+                            "class_name": "DTypePolicy",
+                            "config": {
+                                "name": "int32"
+                            },
+                            "registered_name": null
+                        },
+                        "config_file": "tokenizer.json",
+                        "sequence_length": null,
+                        "add_prefix_space": false,
+                        "unsplittable_tokens": [
+                            "</tool_call>",
+                            "<|endoftext|>",
+                            "<|video_pad|>",
+                            "<think>",
+                            "<|im_end|>",
+                            "</think>",
+                            "<|file_sep|>",
+                            "<tool_call>",
+                            "<|object_ref_end|>",
+                            "<|fim_suffix|>",
+                            "<|quad_start|>",
+                            "<|image_pad|>",
+                            "<|object_ref_start|>",
+                            "<|im_start|>",
+                            "<|box_start|>",
+                            "<|fim_middle|>",
+                            "<|vision_end|>",
+                            "<|fim_pad|>",
+                            "<|box_end|>",
+                            "</tool_response>",
+                            "<|quad_end|>",
+                            "<|vision_pad|>",
+                            "<|fim_prefix|>",
+                            "<|vision_start|>",
+                            "<tool_response>",
+                            "<|repo_name|>"
+                        ]
+                    },
+                    "registered_name": "keras_hub>Qwen3MoeTokenizer"
+                },
+                "config_file": "preprocessor.json",
+                "sequence_length": 1024,
+                "add_start_token": true,
+                "add_end_token": true
+            },
+            "registered_name": "keras_hub>Qwen3MoeCausalLMPreprocessor"
+        },
+        "name": "qwen3_moe_causal_lm"
+    },
+    "registered_name": "keras_hub>Qwen3MoeCausalLM"
+}

tokenizer.json

@@ -0,0 +1,48 @@
+{
+    "module": "keras_hub.src.models.qwen3_moe.qwen3_moe_tokenizer",
+    "class_name": "Qwen3MoeTokenizer",
+    "config": {
+        "name": "qwen3_moe_tokenizer",
+        "trainable": true,
+        "dtype": {
+            "module": "keras",
+            "class_name": "DTypePolicy",
+            "config": {
+                "name": "int32"
+            },
+            "registered_name": null
+        },
+        "config_file": "tokenizer.json",
+        "sequence_length": null,
+        "add_prefix_space": false,
+        "unsplittable_tokens": [
+            "</tool_call>",
+            "<|endoftext|>",
+            "<|video_pad|>",
+            "<think>",
+            "<|im_end|>",
+            "</think>",
+            "<|file_sep|>",
+            "<tool_call>",
+            "<|object_ref_end|>",
+            "<|fim_suffix|>",
+            "<|quad_start|>",
+            "<|image_pad|>",
+            "<|object_ref_start|>",
+            "<|im_start|>",
+            "<|box_start|>",
+            "<|fim_middle|>",
+            "<|vision_end|>",
+            "<|fim_pad|>",
+            "<|box_end|>",
+            "</tool_response>",
+            "<|quad_end|>",
+            "<|vision_pad|>",
+            "<|fim_prefix|>",
+            "<|vision_start|>",
+            "<tool_response>",
+            "<|repo_name|>"
+        ]
+    },
+    "registered_name": "keras_hub>Qwen3MoeTokenizer"
+}