Add files using upload-large-folder tool

config.json

@@ -0,0 +1,58 @@
+{
+    "architectures": [
+        "LLaDA2MoeModelLM"
+    ],
+    "attention_dropout": 0.0,
+    "auto_map": {
+        "AutoConfig": "configuration_llada2_moe.LLaDA2MoeConfig",
+        "AutoModel": "modeling_llada2_moe.LLaDA2MoeModel",
+        "AutoModelForCausalLM": "modeling_llada2_moe.LLaDA2MoeModelLM"
+    },
+    "num_hidden_layers": 32,
+    "hidden_size": 4096,
+    "intermediate_size": 9216,
+    "first_k_dense_replace": 1,
+    "hidden_act": "silu",
+    "max_position_embeddings": 131072,
+    "model_type": "llada2_moe",
+    "moe_intermediate_size": 1024,
+    "norm_topk_prob": true,
+    "num_experts_per_tok": 8,
+    "expert_capacity": 48,
+    "block_size": 32,
+    "norm_head": false,
+    "num_attention_heads": 32,
+    "num_experts": 256,
+    "num_key_value_heads": 4,
+    "rope_theta": 3000000,
+    "rope_scaling": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.52.3",
+    "use_bias": false,
+    "use_rmsnorm": true,
+    "rms_norm_eps": 1e-06,
+    "head_dim": 128,
+    "num_shared_experts": 1,
+    "use_cache": false,
+    "use_qk_norm": true,
+    "use_qkv_bias": false,
+    "embedding_dropout": 0.0,
+    "norm_softmax": false,
+    "output_dropout": 0.0,
+    "vocab_size": 157184,
+    "rotary_dim": 64,
+    "using_split_qkv_in_self_attention": false,
+    "router_dtype": "fp32",
+    "moe_router_enable_expert_bias": true,
+    "routed_scaling_factor": 2.5,
+    "n_group": 8,
+    "topk_group": 4,
+    "score_function": "sigmoid",
+    "initializer_range": 0.02,
+    "max_window_layers": 28,
+    "output_router_logits": false,
+    "pad_token_id": 156892,
+    "partial_rotary_factor": 0.5,
+    "use_sliding_window": false
+}

configuration_llada2_moe.py

@@ -0,0 +1,94 @@
+"""LLaDA2 MoE model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class LLaDA2MoeConfig(PretrainedConfig):
+    model_type = "llada2_moe"
+
+    def __init__(
+        self,
+        vocab_size=30592,
+        hidden_size=1024,
+        intermediate_size=None,
+        num_hidden_layers=24,
+        num_attention_heads=16,
+        num_key_value_heads=0,
+        hidden_act="silu",
+        use_qkv_bias=False,  # llada2 only
+        use_qk_norm=False,
+        use_bias=True,  # llada2 only
+        rms_norm_eps=1e-05,
+        norm_head=False,  # llada2 only
+        tie_word_embeddings=False,  # PretrainedConfig key, here change default value.
+        embedding_dropout=0.1,
+        attention_dropout=0.1,
+        output_dropout=0.1,
+        initializer_range=0.02,
+        max_position_embeddings=16384,
+        rope_theta=10000.0,
+        use_cache=True,
+        use_sliding_window=False,
+        sliding_window=4096,
+        max_window_layers=28,
+        rope_scaling=None,
+        pad_token_id=126081,
+        num_experts=16,
+        num_shared_experts=0,
+        num_experts_per_tok=2,
+        n_group=8,
+        topk_group=4,
+        routed_scaling_factor=2.5,
+        moe_intermediate_size=None,
+        first_k_dense_replace=0,
+        head_dim=None,
+        output_router_logits=False,
+        partial_rotary_factor=0.5,
+        expert_capacity=48,
+        block_size=32,
+        **kwargs,
+    ):
+        self.num_hidden_layers = num_hidden_layers
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.use_qkv_bias = use_qkv_bias
+        self.use_qk_norm = use_qk_norm
+        self.use_bias = use_bias
+        self.norm_head = norm_head
+        self.rms_norm_eps = rms_norm_eps
+        self.embedding_dropout = embedding_dropout
+        self.attention_dropout = attention_dropout
+        self.output_dropout = output_dropout
+        self.initializer_range = initializer_range
+        self.max_position_embeddings = max_position_embeddings
+        self.rope_theta = rope_theta
+        self.use_cache = use_cache
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window
+        self.max_window_layers = max_window_layers
+        self.head_dim = head_dim or self.hidden_size // self.num_attention_heads
+        self.rope_scaling = rope_scaling
+
+        # MoE configs
+        self.num_experts = num_experts
+        self.num_shared_experts = num_shared_experts
+        self.num_experts_per_tok = num_experts_per_tok
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.moe_intermediate_size = moe_intermediate_size
+        self.first_k_dense_replace = first_k_dense_replace
+        self.output_router_logits = output_router_logits
+        self.routed_scaling_factor = routed_scaling_factor
+        self.partial_rotary_factor = partial_rotary_factor
+
+        # Block routing configs
+        self.expert_capacity = expert_capacity
+        self.block_size = block_size
+
+        super().__init__(
+            pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
+        )

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [156892, 156900],
+  "pad_token_id": 156892,
+  "transformers_version": "4.52.3",
+  "use_cache": false
+}

model-00000-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d1990021fcc7bdbd07c1c0089073e2e6c412e1fd8cc4e40d349846087546127
+size 12304705856

model-00001-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4abb1bc2dff14189ad2ed4b9800af3fb54757a0759ce68aed6204b11ee4a7deb
+size 6241216968

model-00002-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5a69255edc9ec0541f84448cb260da5b5067b45709554ea3e2a450b52eb267a
+size 6241217160

model-00003-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d42ed754dbe4f71a8be830e36e60f896dcf3b843515cd02f44ba99c48b798b94
+size 6241217160

model-00004-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc808529f723a2094b0ed4feb6f73eaa57b41f6e3fc808e953890b9b1581b1ad
+size 6241217160

model-00005-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9b716b15ee2052eb27f367add5c058c2cdc379b4c0d694442320afa1f6d8014
+size 6241217160

model-00006-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ef0ab4f979996160dcedb3ea801ddaff6c76a8bd90743331eda8a38c7a37d66
+size 6241217160

model-00007-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ef1088e751d6f96b611596f82c2a75d40f2dc2ebcc89710cba0694aa44e11b4
+size 6241217160

model-00008-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3b29011b004955d811d126e6079b0a020fad605dfb838d994746fc2d3697386
+size 6241217160

model-00009-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec94d969a5d31c9263d2f6ea883dee0e23d824bad7f5df6ce3c066541628051
+size 6241217160

model-00010-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edeb684cb0a29468d3292dd79bb349b8eb8b6b2b455da89b31272cfb02abe43d
+size 6241217160

model-00011-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057bbb40f925141c459c22b34fb29aa43293f9686b99c52eff40a5790d3b97a4
+size 6241217160

model-00012-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5297203652203933efb0e5c1b4231592cd387d3e6a8ba005d6ad718ee1ba5cd5
+size 6241217528

model-00013-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b52c6ceb64ce9cc192e7153c1003f58feb84a6b4eff66dd8bcf14cbeaca08c3
+size 6241217904

model-00014-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:790c089f45a061711ff6e2442359702b3011e4cf1055928e53175ff30db7da6c
+size 6241217904

model-00015-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1688f8d99143a104fc743959d293c9c7e8ed88ca9246a32b6c66e568250e0e76
+size 6241217904

model-00016-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b673b90d7ce20df76bcd63ed68da1fc56a23c5b6ea66d80003a6b484a8583cb
+size 6241217904

model-00017-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9741de327958cd1542206f46df66c064ae8b55eaf6408c57ec32099097dd22e
+size 6241217904

model-00018-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2b1eedc53f6f5110c9f7cbda3e39682600572aa142b1c8792a50300572336ae
+size 6241217904

model-00019-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ce6cc65fadbe15d0d9695e1a5e371f2df4ae2c6bcb5fcb93ed731a3f4322fc2
+size 6241217904

model-00020-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a173a41b95409b6d1aff76f2c923f51936f9679c07b4480e3220ed5e8aaf6688
+size 6241217904

model-00021-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f369627b48e8ff61d77daca5dddc4fe8e66428156d25030d3c45c522fa212f48
+size 6241217904

model-00022-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1670d3b111e173e45f773e2aef2a982f7e8b074c5a74ce2b9d8a2d795d95a30
+size 6241217904

model-00023-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cb566ee0a2bb9161946dd7dd2785f3ff35f31cd59d26b8e52cd8888bc33126e
+size 6241217904

model-00024-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:110d4e1c4071796e4e6d3562a6afa55a5e9653f79b22d507c9472baea9ab8a0e
+size 6241217904

model-00025-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e50d758bed684abfd4dfe567d41d652908bb809ad4581925d750d4f82b384e12
+size 6241217904

model-00026-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12c25ff4880833827155b08ead89b59ee66cbf1942c7f30a511d6403df411be6
+size 6241217904

model-00027-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:332260d4dd6c96a68cc679c712032caa5dda9ed6170f0b38b98917428ca1aa7d
+size 6241217904

model-00028-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d67a39cce185e1c5ae88efe66247b94231de6ad4dd43d9696fd2a517a56a0ea2
+size 6241217904

model-00029-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:300e136169bba055b66c2c47602494a4457e2c3f06764f2b7ba17cbaa9599e7f
+size 6241217904

model-00030-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dd9ec5fdd4b8bb609de7cc6c6dc3b5b18af98df086d6bc3c2da678f6d81fe78
+size 6241217904

model-00031-of-00032.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c133427faaa6fc619549824fd96f60b2a1901eca7a0c47c85f8021a6168c3a7
+size 6241217904

special_tokens_map.json

@@ -0,0 +1,8 @@
+{
+  "bos_token": "<|startoftext|>",
+  "cls_token": "[CLS]",
+  "eos_token": "<|endoftext|>",
+  "gmask_token": "[gMASK]",
+  "pad_token": "<|endoftext|>",
+  "mask_token": "<|mask|>"
+}

tokenizer_config.json

@@ -0,0 +1,145 @@
+{
+  "added_tokens_decoder": {
+    "156891": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156892": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156893": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156894": {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156895": {
+      "content": "<|mask|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156896": {
+      "content": "<tools>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156897": {
+      "content": "</tools>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156898": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156899": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156900": {
+      "content": "<|role_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156901": {
+      "content": "<|tool_calls_section_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156902": {
+      "content": "<|tool_calls_section_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156903": {
+      "content": "<|tool_call_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156904": {
+      "content": "<|tool_call_argument_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156905": {
+      "content": "<|tool_call_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "eos_token": "<|endoftext|>",
+  "mask_token": "<|mask|>",
+  "fast_tokenizer": true,
+  "gmask_token": "[gMASK]",
+  "merges_file": null,
+  "model_max_length": 262144,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CustomFastTokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_llada2.CustomFastTokenizer",
+      null
+    ]
+  },
+  "trust_remote_code": true,
+  "vocab_file": null
+}