Hugging Face's logo

TheMindExpansionNetwork
/
mindbot-flash-v1

Safetensors
glm4_moe_lite
Model card Files
xet
 Community
TheMindExpansionNetwork commited on
Commit
5091fe3
·
verified ·
1 Parent(s): f89eefb

Upload folder using huggingface_hub

Browse files
Files changed (17) hide show
  1. config.json +104 -0
  2. generation_config.json +11 -0
  3. model-00001-of-00014.safetensors +3 -0
  4. model-00002-of-00014.safetensors +3 -0
  5. model-00003-of-00014.safetensors +3 -0
  6. model-00004-of-00014.safetensors +3 -0
  7. model-00005-of-00014.safetensors +3 -0
  8. model-00006-of-00014.safetensors +3 -0
  9. model-00007-of-00014.safetensors +3 -0
  10. model-00008-of-00014.safetensors +3 -0
  11. model-00009-of-00014.safetensors +3 -0
  12. model-00010-of-00014.safetensors +3 -0
  13. model-00011-of-00014.safetensors +3 -0
  14. model-00012-of-00014.safetensors +3 -0
  15. model-00013-of-00014.safetensors +3 -0
  16. model-00014-of-00014.safetensors +3 -0
  17. model.safetensors.index.json +0 -0
config.json ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Glm4MoeLiteForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 0,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": [
10
+ 154820,
11
+ 154827,
12
+ 154829
13
+ ],
14
+ "first_k_dense_replace": 1,
15
+ "head_dim": 64,
16
+ "hidden_act": "silu",
17
+ "hidden_size": 2048,
18
+ "initializer_range": 0.02,
19
+ "intermediate_size": 10240,
20
+ "kv_lora_rank": 512,
21
+ "max_position_embeddings": 202752,
22
+ "mlp_layer_types": [
23
+ "dense",
24
+ "sparse",
25
+ "sparse",
26
+ "sparse",
27
+ "sparse",
28
+ "sparse",
29
+ "sparse",
30
+ "sparse",
31
+ "sparse",
32
+ "sparse",
33
+ "sparse",
34
+ "sparse",
35
+ "sparse",
36
+ "sparse",
37
+ "sparse",
38
+ "sparse",
39
+ "sparse",
40
+ "sparse",
41
+ "sparse",
42
+ "sparse",
43
+ "sparse",
44
+ "sparse",
45
+ "sparse",
46
+ "sparse",
47
+ "sparse",
48
+ "sparse",
49
+ "sparse",
50
+ "sparse",
51
+ "sparse",
52
+ "sparse",
53
+ "sparse",
54
+ "sparse",
55
+ "sparse",
56
+ "sparse",
57
+ "sparse",
58
+ "sparse",
59
+ "sparse",
60
+ "sparse",
61
+ "sparse",
62
+ "sparse",
63
+ "sparse",
64
+ "sparse",
65
+ "sparse",
66
+ "sparse",
67
+ "sparse",
68
+ "sparse",
69
+ "sparse"
70
+ ],
71
+ "model_type": "glm4_moe_lite",
72
+ "moe_intermediate_size": 1536,
73
+ "n_group": 1,
74
+ "n_routed_experts": 64,
75
+ "n_shared_experts": 1,
76
+ "norm_topk_prob": true,
77
+ "num_attention_heads": 20,
78
+ "num_experts_per_tok": 4,
79
+ "num_hidden_layers": 47,
80
+ "num_key_value_heads": 20,
81
+ "num_nextn_predict_layers": 1,
82
+ "pad_token_id": 154820,
83
+ "partial_rotary_factor": 1.0,
84
+ "pretraining_tp": 1,
85
+ "q_lora_rank": 768,
86
+ "qk_head_dim": 256,
87
+ "qk_nope_head_dim": 192,
88
+ "qk_rope_head_dim": 64,
89
+ "rms_norm_eps": 1e-05,
90
+ "rope_interleave": true,
91
+ "rope_parameters": {
92
+ "partial_rotary_factor": 1.0,
93
+ "rope_theta": 1000000,
94
+ "rope_type": "default"
95
+ },
96
+ "routed_scaling_factor": 1.8,
97
+ "tie_word_embeddings": false,
98
+ "topk_group": 1,
99
+ "topk_method": "noaux_tc",
100
+ "transformers_version": "5.0.0",
101
+ "use_cache": true,
102
+ "v_head_dim": 256,
103
+ "vocab_size": 154880
104
+ }
generation_config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": [
4
+ 154820,
5
+ 154827,
6
+ 154829
7
+ ],
8
+ "pad_token_id": 154820,
9
+ "temperature": 1.0,
10
+ "transformers_version": "5.0.0"
11
+ }
model-00001-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f383b2dd674da0b14d6c5f48419b8080bf82cc0079488a9c80c57ebeeb016479
3
+ size 4382096496
model-00002-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4bce98e99f79b74c506a8f9168c09b69b1bcb3dee23de8dbd1ade04c7440f6e
3
+ size 4679931984
model-00003-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:336d0d969b9aed45962cfe415ec7e5f8beb7487e4ba402b2a613680336f584a9
3
+ size 4214606024
model-00004-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef2e0f2354d79139a4aad09758a24a79013ae36c4f47efb826228775be9c09ea
3
+ size 4679932744
model-00005-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6b127dab8445365a0df6eb1f8c33473b4f8908c0dccdf7024245d43e6d6d98f
3
+ size 4214606648
model-00006-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7846aa20f0a5e6b846d109e68b32af7bc257a7ddfbe07007ccc086c6e39ac75f
3
+ size 4679932744
model-00007-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07ef8e8501dc55603b9ccc442dd63ae70cc54b6c98aff7b9e3a46e38e4b913da
3
+ size 4214606648
model-00008-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2eb47e2652748a40eec2de3d56ced80042e3f621a1a4558ca079f36e3416cd6c
3
+ size 4679932744
model-00009-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7701b7191106000b1ce36e1e56a749d27d4010923c15153b80c6ad06a640bcfb
3
+ size 4214606648
model-00010-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f64edeeb416cea775860d2074ed22d54492ce53d87206b3b535d603a269268e1
3
+ size 4679932744
model-00011-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b6f018c8fa75d1d06ccf81d3e32fad967a841d1775a53e451a6ad191e2ccfe6
3
+ size 4214606648
model-00012-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb352d3aa9064f5481d1015a18fa110578dd6bbd6f3897f0d3517a768433108f
3
+ size 4679932744
model-00013-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:412ca6771ccca7e21caae9c0daf2b408f056dfdca3c3669b06220bbb6dd17669
3
+ size 4214606648
model-00014-of-00014.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdab70082571ba1eb9b04eeccf9149bdf1498def76465f706977d8e9933bd7c6
3
+ size 2138635568
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff