Hugging Face's logo

RichardErkhov
/
yhavinga_-_Boreas-7B-chat-4bits

Text Generation
Transformers
Safetensors
mistral
conversational
text-generation-inference
4-bit precision
bitsandbytes
Model card Files
xet
 Community
RichardErkhov commited on
Commit
c196463
·
verified ·
1 Parent(s): 8746267

uploaded model

Browse files
Files changed (1) hide show
  1. config.json +150 -0
config.json ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Boreas-7B-chat",
3
+ "architectures": [
4
+ "MistralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "attention_partition_spec": [
9
+ [
10
+ "dp",
11
+ "fsdp"
12
+ ],
13
+ "sp",
14
+ "tp",
15
+ null
16
+ ],
17
+ "attn_mechanism": "normal",
18
+ "axis_dims": [
19
+ 1,
20
+ -1,
21
+ 1,
22
+ 1
23
+ ],
24
+ "axis_names": [
25
+ "dp",
26
+ "fsdp",
27
+ "tp",
28
+ "sp"
29
+ ],
30
+ "backend": null,
31
+ "bias_partition_spec": [
32
+ [
33
+ "dp",
34
+ "fsdp"
35
+ ],
36
+ null,
37
+ null,
38
+ null
39
+ ],
40
+ "bits": null,
41
+ "block_b": 1,
42
+ "block_k": 128,
43
+ "block_k_dkv": 128,
44
+ "block_k_dq": 128,
45
+ "block_k_major": 128,
46
+ "block_k_major_dkv": 128,
47
+ "block_k_major_dq": 128,
48
+ "block_q": 128,
49
+ "block_q_dkv": 128,
50
+ "block_q_dq": 128,
51
+ "block_q_major_dkv": 128,
52
+ "bos_token_id": 1,
53
+ "easy_method": "train",
54
+ "eos_token_id": 2,
55
+ "generation_attention_partition_spec": [
56
+ [
57
+ "dp",
58
+ "fsdp"
59
+ ],
60
+ null,
61
+ "tp",
62
+ null
63
+ ],
64
+ "generation_bias_partition_spec": [
65
+ [
66
+ "dp",
67
+ "fsdp"
68
+ ],
69
+ null,
70
+ null,
71
+ null
72
+ ],
73
+ "generation_query_partition_spec": [
74
+ [
75
+ "dp",
76
+ "fsdp"
77
+ ],
78
+ "sp",
79
+ null,
80
+ null
81
+ ],
82
+ "hidden_act": "silu",
83
+ "hidden_size": 4096,
84
+ "initializer_range": 0.02,
85
+ "intermediate_size": 14336,
86
+ "key_partition_spec": [
87
+ [
88
+ "dp",
89
+ "fsdp"
90
+ ],
91
+ "sp",
92
+ "tp",
93
+ null
94
+ ],
95
+ "max_position_embeddings": 32768,
96
+ "model_type": "mistral",
97
+ "num_attention_heads": 32,
98
+ "num_hidden_layers": 32,
99
+ "num_key_value_heads": 8,
100
+ "number_rep_kv": 1,
101
+ "quantization_config": {
102
+ "_load_in_4bit": true,
103
+ "_load_in_8bit": false,
104
+ "bnb_4bit_compute_dtype": "float32",
105
+ "bnb_4bit_quant_storage": "uint8",
106
+ "bnb_4bit_quant_type": "fp4",
107
+ "bnb_4bit_use_double_quant": false,
108
+ "llm_int8_enable_fp32_cpu_offload": false,
109
+ "llm_int8_has_fp16_weight": false,
110
+ "llm_int8_skip_modules": null,
111
+ "llm_int8_threshold": 6.0,
112
+ "load_in_4bit": true,
113
+ "load_in_8bit": false,
114
+ "quant_method": "bitsandbytes"
115
+ },
116
+ "query_partition_spec": [
117
+ [
118
+ "dp",
119
+ "fsdp"
120
+ ],
121
+ "sp",
122
+ "tp",
123
+ null
124
+ ],
125
+ "rms_norm_eps": 1e-05,
126
+ "rope_scaling": null,
127
+ "rope_theta": 10000.0,
128
+ "scan_attention_layers": false,
129
+ "scan_mlp_chunk_size": 1024,
130
+ "scan_ring_attention": false,
131
+ "shard_attention_computation": true,
132
+ "sliding_window": 4096,
133
+ "tie_word_embeddings": false,
134
+ "torch_dtype": "float16",
135
+ "transformers_version": "4.39.3",
136
+ "use_cache": true,
137
+ "use_scan_mlp": false,
138
+ "use_sharded_kv_caching": true,
139
+ "use_sharding_constraint": false,
140
+ "value_partition_spec": [
141
+ [
142
+ "dp",
143
+ "fsdp"
144
+ ],
145
+ "sp",
146
+ "tp",
147
+ null
148
+ ],
149
+ "vocab_size": 32000
150
+ }