Hugging Face's logo

Yi30
/
test-llmc

qwen3_moe
compressed-tensors
Model card Files
xet
Community
Yi30 commited on
Commit
3c08e37
·
verified ·
1 Parent(s): d2eddd6

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +123 -0
config.json ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "decoder_sparse_step": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 151645,
11
+ "head_dim": 128,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 2048,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 6144,
16
+ "max_position_embeddings": 40960,
17
+ "max_window_layers": 48,
18
+ "mlp_only_layers": [],
19
+ "model_type": "qwen3_moe",
20
+ "moe_intermediate_size": 768,
21
+ "norm_topk_prob": true,
22
+ "num_attention_heads": 32,
23
+ "num_experts": 128,
24
+ "num_experts_per_tok": 8,
25
+ "num_hidden_layers": 48,
26
+ "num_key_value_heads": 4,
27
+ "output_router_logits": false,
28
+ "quantization_config": {
29
+ "config_groups": {
30
+ "group_0": {
31
+ "format": "pack-quantized",
32
+ "input_activations": null,
33
+ "output_activations": null,
34
+ "targets": [
35
+ "Linear"
36
+ ],
37
+ "weights": {
38
+ "actorder": null,
39
+ "block_structure": null,
40
+ "dynamic": false,
41
+ "group_size": 128,
42
+ "num_bits": 4,
43
+ "observer": "minmax",
44
+ "observer_kwargs": {},
45
+ "scale_dtype": null,
46
+ "strategy": "group",
47
+ "symmetric": true,
48
+ "type": "int",
49
+ "zp_dtype": null
50
+ }
51
+ }
52
+ },
53
+ "format": "pack-quantized",
54
+ "global_compression_ratio": null,
55
+ "ignore": [
56
+ "model.layers.0.mlp.gate",
57
+ "model.layers.1.mlp.gate",
58
+ "model.layers.2.mlp.gate",
59
+ "model.layers.3.mlp.gate",
60
+ "model.layers.4.mlp.gate",
61
+ "model.layers.5.mlp.gate",
62
+ "model.layers.6.mlp.gate",
63
+ "model.layers.7.mlp.gate",
64
+ "model.layers.8.mlp.gate",
65
+ "model.layers.9.mlp.gate",
66
+ "model.layers.10.mlp.gate",
67
+ "model.layers.11.mlp.gate",
68
+ "model.layers.12.mlp.gate",
69
+ "model.layers.13.mlp.gate",
70
+ "model.layers.14.mlp.gate",
71
+ "model.layers.15.mlp.gate",
72
+ "model.layers.16.mlp.gate",
73
+ "model.layers.17.mlp.gate",
74
+ "model.layers.18.mlp.gate",
75
+ "model.layers.19.mlp.gate",
76
+ "model.layers.20.mlp.gate",
77
+ "model.layers.21.mlp.gate",
78
+ "model.layers.22.mlp.gate",
79
+ "model.layers.23.mlp.gate",
80
+ "model.layers.24.mlp.gate",
81
+ "model.layers.25.mlp.gate",
82
+ "model.layers.26.mlp.gate",
83
+ "model.layers.27.mlp.gate",
84
+ "model.layers.28.mlp.gate",
85
+ "model.layers.29.mlp.gate",
86
+ "model.layers.30.mlp.gate",
87
+ "model.layers.31.mlp.gate",
88
+ "model.layers.32.mlp.gate",
89
+ "model.layers.33.mlp.gate",
90
+ "model.layers.34.mlp.gate",
91
+ "model.layers.35.mlp.gate",
92
+ "model.layers.36.mlp.gate",
93
+ "model.layers.37.mlp.gate",
94
+ "model.layers.38.mlp.gate",
95
+ "model.layers.39.mlp.gate",
96
+ "model.layers.40.mlp.gate",
97
+ "model.layers.41.mlp.gate",
98
+ "model.layers.42.mlp.gate",
99
+ "model.layers.43.mlp.gate",
100
+ "model.layers.44.mlp.gate",
101
+ "model.layers.45.mlp.gate",
102
+ "model.layers.46.mlp.gate",
103
+ "model.layers.47.mlp.gate",
104
+ "lm_head"
105
+ ],
106
+ "kv_cache_scheme": null,
107
+ "quant_method": "compressed-tensors",
108
+ "quantization_status": "compressed",
109
+ "sparsity_config": {},
110
+ "transform_config": {},
111
+ "version": "0.12.3.a20251214"
112
+ },
113
+ "rms_norm_eps": 1e-06,
114
+ "rope_scaling": null,
115
+ "rope_theta": 1000000.0,
116
+ "router_aux_loss_coef": 0.001,
117
+ "sliding_window": null,
118
+ "tie_word_embeddings": false,
119
+ "transformers_version": "4.57.3",
120
+ "use_cache": true,
121
+ "use_sliding_window": false,
122
+ "vocab_size": 151936
123
+ }