File size: 5,462 Bytes
49caa1e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 |
{
"architectures": [
"MiniCPMForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_minicpm.MiniCPMConfig",
"AutoModel": "modeling_minicpm.MiniCPMModel",
"AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
"AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
"AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
},
"bos_token_id": 1,
"dim_model_base": 256,
"eos_token_id": [
2,
73440
],
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.1,
"intermediate_size": 16384,
"max_position_embeddings": 65536,
"model_type": "minicpm",
"mup_denominator": 32,
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 2,
"pad_token_id": 2,
"quantization": {
"group_size": 64,
"bits": 8
},
"quantization_config": {
"group_size": 64,
"bits": 8
},
"rms_norm_eps": 1e-06,
"rope_scaling": {
"rope_type": "longrope",
"long_factor": [
0.9982316082870437,
1.033048153422584,
1.0749920956484724,
1.1255096879436193,
1.1863348602111476,
1.259543828902579,
1.3476188888731149,
1.4535223827776373,
1.5807816745852985,
1.7335856049489526,
1.9168922912975785,
2.1365471404135326,
2.3994084200118646,
2.713475511863602,
3.0880118452194134,
3.533650295140154,
4.062463396503134,
4.687974098908333,
5.425075306704039,
6.289818967956352,
7.29902962722721,
8.469695779093664,
9.81809877306655,
11.358657902065282,
13.102505860712087,
15.055862949967128,
17.218348131364184,
19.581439255386453,
22.127353314656723,
24.828633849376587,
27.6486820771775,
30.54334096108829,
33.46345345363812,
36.358112337548896,
39.17816056534983,
41.879441100069684,
44.425355159339965,
46.78844628336223,
48.95093146475928,
50.90428855401433,
52.648136512661125,
54.18869564165987,
55.537098635632745,
56.7077647874992,
57.71697544677006,
58.58171910802236,
59.31882031581807,
59.94433101822328,
60.47314411958625,
60.918782569507,
61.29331890286281,
61.60738599471455,
61.87024727431288,
62.089902123428836,
62.27320880977746,
62.42601274014111,
62.55327203194878,
62.65917552585329,
62.74725058582382,
62.82045955451526,
62.88128472678279,
62.931802319077946,
62.97374626130382,
63.008562806439365
],
"short_factor": [
0.9982316082870437,
1.033048153422584,
1.0749920956484724,
1.1255096879436193,
1.1863348602111476,
1.259543828902579,
1.3476188888731149,
1.4535223827776373,
1.5807816745852985,
1.7335856049489526,
1.9168922912975785,
2.1365471404135326,
2.3994084200118646,
2.713475511863602,
3.0880118452194134,
3.533650295140154,
4.062463396503134,
4.687974098908333,
5.425075306704039,
6.289818967956352,
7.29902962722721,
8.469695779093664,
9.81809877306655,
11.358657902065282,
13.102505860712087,
15.055862949967128,
17.218348131364184,
19.581439255386453,
22.127353314656723,
24.828633849376587,
27.6486820771775,
30.54334096108829,
33.46345345363812,
36.358112337548896,
39.17816056534983,
41.879441100069684,
44.425355159339965,
46.78844628336223,
48.95093146475928,
50.90428855401433,
52.648136512661125,
54.18869564165987,
55.537098635632745,
56.7077647874992,
57.71697544677006,
58.58171910802236,
59.31882031581807,
59.94433101822328,
60.47314411958625,
60.918782569507,
61.29331890286281,
61.60738599471455,
61.87024727431288,
62.089902123428836,
62.27320880977746,
62.42601274014111,
62.55327203194878,
62.65917552585329,
62.74725058582382,
62.82045955451526,
62.88128472678279,
62.931802319077946,
62.97374626130382,
63.008562806439365
],
"original_max_position_embeddings": 65536
},
"rope_theta": 10000.0,
"scale_depth": 1.4,
"scale_emb": 12,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.3",
"use_cache": true,
"vocab_size": 73448
} |