Upload folder using huggingface_hub

chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|' + message['role'] + '|>' + '
+' + message['content'] + '<|end|>
+' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>
+' }}{% else %}{{ eos_token }}{% endif %}

config.json

@@ -0,0 +1,366 @@
+{
+  "architectures": [
+    "PhimoeForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_slimmoe.PhiMoEConfig",
+    "AutoModelForCausalLM": "modeling_slimmoe.PhiMoEForCausalLM"
+  },
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 32000,
+  "expert_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "input_jitter_noise": 0.01,
+  "intermediate_size": 960,
+  "is_pruned": true,
+  "lm_head_bias": true,
+  "max_position_embeddings": 4096,
+  "model_type": "phimoe",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 48,
+  "old_to_new_mapping": {
+    "0": {
+      "12": 3,
+      "14": 4,
+      "4": 0,
+      "5": 1,
+      "8": 2
+    },
+    "1": {
+      "1": 0,
+      "10": 6,
+      "12": 7,
+      "14": 8,
+      "3": 1,
+      "4": 2,
+      "5": 3,
+      "7": 4,
+      "8": 5
+    },
+    "10": {
+      "1": 0,
+      "13": 6,
+      "15": 7,
+      "2": 1,
+      "4": 2,
+      "6": 3,
+      "7": 4,
+      "8": 5
+    },
+    "11": {
+      "0": 0,
+      "11": 3,
+      "12": 4,
+      "14": 5,
+      "15": 6,
+      "6": 1,
+      "8": 2
+    },
+    "12": {
+      "0": 0,
+      "10": 3,
+      "11": 4,
+      "12": 5,
+      "14": 6,
+      "15": 7,
+      "5": 1,
+      "6": 2
+    },
+    "13": {
+      "11": 2,
+      "13": 3,
+      "14": 4,
+      "2": 0,
+      "3": 1
+    },
+    "14": {
+      "11": 2,
+      "12": 3,
+      "13": 4,
+      "14": 5,
+      "2": 0,
+      "3": 1
+    },
+    "15": {
+      "0": 0,
+      "1": 1,
+      "10": 5,
+      "12": 6,
+      "14": 7,
+      "3": 2,
+      "4": 3,
+      "7": 4
+    },
+    "16": {
+      "1": 0,
+      "11": 5,
+      "12": 6,
+      "14": 7,
+      "2": 1,
+      "3": 2,
+      "5": 3,
+      "7": 4
+    },
+    "17": {
+      "1": 0,
+      "12": 7,
+      "14": 8,
+      "2": 1,
+      "3": 2,
+      "4": 3,
+      "5": 4,
+      "6": 5,
+      "7": 6
+    },
+    "18": {
+      "10": 6,
+      "12": 7,
+      "14": 8,
+      "3": 0,
+      "4": 1,
+      "5": 2,
+      "7": 3,
+      "8": 4,
+      "9": 5
+    },
+    "19": {
+      "0": 0,
+      "10": 6,
+      "14": 7,
+      "3": 1,
+      "4": 2,
+      "5": 3,
+      "6": 4,
+      "7": 5
+    },
+    "2": {
+      "1": 0,
+      "10": 8,
+      "11": 9,
+      "12": 10,
+      "13": 11,
+      "14": 12,
+      "15": 13,
+      "2": 1,
+      "3": 2,
+      "4": 3,
+      "5": 4,
+      "6": 5,
+      "7": 6,
+      "8": 7
+    },
+    "20": {
+      "1": 0,
+      "12": 5,
+      "13": 6,
+      "14": 7,
+      "15": 8,
+      "2": 1,
+      "5": 2,
+      "6": 3,
+      "7": 4
+    },
+    "21": {
+      "14": 3,
+      "15": 4,
+      "4": 0,
+      "6": 1,
+      "9": 2
+    },
+    "22": {
+      "0": 0,
+      "1": 1,
+      "14": 7,
+      "15": 8,
+      "3": 2,
+      "4": 3,
+      "6": 4,
+      "8": 5,
+      "9": 6
+    },
+    "23": {
+      "0": 0,
+      "10": 5,
+      "12": 6,
+      "14": 7,
+      "2": 1,
+      "3": 2,
+      "5": 3,
+      "9": 4
+    },
+    "24": {
+      "1": 0,
+      "11": 5,
+      "14": 6,
+      "3": 1,
+      "5": 2,
+      "7": 3,
+      "9": 4
+    },
+    "25": {
+      "0": 0,
+      "10": 5,
+      "11": 6,
+      "12": 7,
+      "15": 8,
+      "2": 1,
+      "4": 2,
+      "8": 3,
+      "9": 4
+    },
+    "26": {
+      "11": 4,
+      "12": 5,
+      "15": 6,
+      "2": 0,
+      "3": 1,
+      "4": 2,
+      "8": 3
+    },
+    "27": {
+      "0": 0,
+      "10": 5,
+      "11": 6,
+      "12": 7,
+      "13": 8,
+      "15": 9,
+      "2": 1,
+      "4": 2,
+      "5": 3,
+      "9": 4
+    },
+    "28": {
+      "1": 0,
+      "10": 5,
+      "11": 6,
+      "12": 7,
+      "13": 8,
+      "4": 1,
+      "6": 2,
+      "7": 3,
+      "9": 4
+    },
+    "29": {
+      "1": 0,
+      "10": 6,
+      "11": 7,
+      "4": 1,
+      "6": 2,
+      "7": 3,
+      "8": 4,
+      "9": 5
+    },
+    "3": {
+      "1": 0,
+      "10": 6,
+      "11": 7,
+      "13": 8,
+      "14": 9,
+      "15": 10,
+      "4": 1,
+      "6": 2,
+      "7": 3,
+      "8": 4,
+      "9": 5
+    },
+    "30": {
+      "1": 0,
+      "10": 4,
+      "11": 5,
+      "12": 6,
+      "15": 7,
+      "3": 1,
+      "7": 2,
+      "9": 3
+    },
+    "31": {
+      "1": 0,
+      "10": 6,
+      "12": 7,
+      "15": 8,
+      "2": 1,
+      "4": 2,
+      "7": 3,
+      "8": 4,
+      "9": 5
+    },
+    "4": {
+      "10": 6,
+      "13": 7,
+      "15": 8,
+      "4": 0,
+      "5": 1,
+      "6": 2,
+      "7": 3,
+      "8": 4,
+      "9": 5
+    },
+    "5": {
+      "12": 3,
+      "13": 4,
+      "15": 5,
+      "4": 0,
+      "7": 1,
+      "9": 2
+    },
+    "6": {
+      "0": 0,
+      "1": 1,
+      "10": 6,
+      "13": 7,
+      "15": 8,
+      "5": 2,
+      "6": 3,
+      "7": 4,
+      "9": 5
+    },
+    "7": {
+      "1": 0,
+      "10": 3,
+      "12": 4,
+      "13": 5,
+      "3": 1,
+      "7": 2
+    },
+    "8": {
+      "1": 0,
+      "10": 7,
+      "12": 8,
+      "14": 9,
+      "3": 1,
+      "4": 2,
+      "5": 3,
+      "6": 4,
+      "7": 5,
+      "8": 6
+    },
+    "9": {
+      "0": 0,
+      "11": 3,
+      "4": 1,
+      "6": 2
+    }
+  },
+  "output_router_logits": false,
+  "pruned": true,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.0,
+  "router_jitter_noise": 0.01,
+  "sliding_window": 2047,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 32064
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.57.3"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c36badc2adb43d332183f747236cab625e595ea69570e21acd2d68373daaa828
+size 4997213700

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae9504c70847ca3d43da4540d5b9324f72a0f62855c59351a30793b301acc185
+size 4290394588

pruning_map.txt

@@ -0,0 +1,33 @@
+Pruned Experts by Original Layer (Target layer, Expert ID):
+  Layer  0: [0, 1, 2, 3, 6, 7, 9, 10, 11, 13, 15]
+  Layer  1: [0, 2, 6, 9, 11, 13, 15]
+  Layer  2: [0, 9]
+  Layer  3: [0, 2, 3, 5, 12]
+  Layer  4: [0, 1, 2, 3, 11, 12, 14]
+  Layer  5: [0, 1, 2, 3, 5, 6, 8, 10, 11, 14]
+  Layer  6: [2, 3, 4, 8, 11, 12, 14]
+  Layer  7: [0, 2, 4, 5, 6, 8, 9, 11, 14, 15]
+  Layer  8: [0, 2, 9, 11, 13, 15]
+  Layer  9: [1, 2, 3, 5, 7, 8, 9, 10, 12, 13, 14, 15]
+  Layer 10: [0, 3, 5, 9, 10, 11, 12, 14]
+  Layer 11: [1, 2, 3, 4, 5, 7, 9, 10, 13]
+  Layer 12: [1, 2, 3, 4, 7, 8, 9, 13]
+  Layer 13: [0, 1, 4, 5, 6, 7, 8, 9, 10, 12, 15]
+  Layer 14: [0, 1, 4, 5, 6, 7, 8, 9, 10, 15]
+  Layer 15: [2, 5, 6, 8, 9, 11, 13, 15]
+  Layer 16: [0, 4, 6, 8, 9, 10, 13, 15]
+  Layer 17: [0, 8, 9, 10, 11, 13, 15]
+  Layer 18: [0, 1, 2, 6, 11, 13, 15]
+  Layer 19: [1, 2, 8, 9, 11, 12, 13, 15]
+  Layer 20: [0, 3, 4, 8, 9, 10, 11]
+  Layer 21: [0, 1, 2, 3, 5, 7, 8, 10, 11, 12, 13]
+  Layer 22: [2, 5, 7, 10, 11, 12, 13]
+  Layer 23: [1, 4, 6, 7, 8, 11, 13, 15]
+  Layer 24: [0, 2, 4, 6, 8, 10, 12, 13, 15]
+  Layer 25: [1, 3, 5, 6, 7, 13, 14]
+  Layer 26: [0, 1, 5, 6, 7, 9, 10, 13, 14]
+  Layer 27: [1, 3, 6, 7, 8, 14]
+  Layer 28: [0, 2, 3, 5, 8, 14, 15]
+  Layer 29: [0, 2, 3, 5, 12, 13, 14, 15]
+  Layer 30: [0, 2, 4, 5, 6, 8, 13, 14]
+  Layer 31: [0, 3, 5, 6, 11, 13, 14]

rexmoe_routers.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70ef9516c7741a22739c75df69e7cdaa4d2e71425f5e9702627c39016997f031
+size 12623346

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,131 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}