keisuke-miyako commited on
Commit
d682567
·
verified ·
1 Parent(s): 606408f

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. config.json +6 -3
  2. model.bin +2 -2
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +135 -29
  5. vocabulary.json +0 -0
config.json CHANGED
@@ -1,7 +1,10 @@
1
  {
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
- "layer_norm_epsilon": 1e-12,
5
- "multi_query_attention": false,
6
- "unk_token": "[UNK]"
 
 
 
7
  }
 
1
  {
2
  "bos_token": "<s>",
3
  "eos_token": "</s>",
4
+ "layer_norm_epsilon": 1e-05,
5
+ "multi_query_attention": true,
6
+ "quantization_bits": null,
7
+ "quantization_group_size": null,
8
+ "quantization_type": 0,
9
+ "unk_token": "<unk>"
10
  }
model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:09905d10594ed99a59f0f4d9dd304b31247c0fe41b1bcd8102320fc3795b10ba
3
- size 66728403
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0837ba41b032f2ec0d030a0fe444657e27e158bf41edf64ddb92237da00f088a
3
+ size 2815093409
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,64 +1,170 @@
1
  {
 
 
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
- "content": "[PAD]",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
8
  "single_word": false,
9
  "special": true
10
  },
11
- "100": {
12
- "content": "[UNK]",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
  },
19
- "101": {
20
- "content": "[CLS]",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
  },
27
- "102": {
28
- "content": "[SEP]",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
  },
35
- "103": {
36
- "content": "[MASK]",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  }
43
  },
44
- "clean_up_tokenization_spaces": true,
45
- "cls_token": "[CLS]",
46
- "do_basic_tokenize": true,
47
- "do_lower_case": true,
48
- "mask_token": "[MASK]",
49
- "max_length": 512,
50
- "model_max_length": 512,
51
- "never_split": null,
52
- "pad_to_multiple_of": null,
53
- "pad_token": "[PAD]",
54
- "pad_token_type_id": 0,
55
- "padding_side": "right",
56
- "sep_token": "[SEP]",
57
- "stride": 0,
58
- "strip_accents": null,
59
- "tokenize_chinese_chars": true,
60
- "tokenizer_class": "BertTokenizer",
61
- "truncation_side": "right",
62
- "truncation_strategy": "longest_first",
63
- "unk_token": "[UNK]"
64
  }
 
1
  {
2
+ "add_bos_token": false,
3
+ "add_dummy_prefix_space": false,
4
+ "add_eos_token": true,
5
+ "add_prefix_space": false,
6
  "added_tokens_decoder": {
7
  "0": {
8
+ "content": "<unk>",
9
  "lstrip": false,
10
  "normalized": false,
11
  "rstrip": false,
12
  "single_word": false,
13
  "special": true
14
  },
15
+ "1": {
16
+ "content": "<s>",
17
  "lstrip": false,
18
  "normalized": false,
19
  "rstrip": false,
20
  "single_word": false,
21
  "special": true
22
  },
23
+ "2": {
24
+ "content": "</s>",
25
  "lstrip": false,
26
  "normalized": false,
27
  "rstrip": false,
28
  "single_word": false,
29
  "special": true
30
  },
31
+ "3": {
32
+ "content": "<pad>",
33
  "lstrip": false,
34
  "normalized": false,
35
  "rstrip": false,
36
  "single_word": false,
37
  "special": true
38
  },
39
+ "4": {
40
+ "content": "<sep>",
41
  "lstrip": false,
42
  "normalized": false,
43
  "rstrip": false,
44
  "single_word": false,
45
  "special": true
46
+ },
47
+ "5": {
48
+ "content": "<mask>",
49
+ "lstrip": false,
50
+ "normalized": false,
51
+ "rstrip": false,
52
+ "single_word": false,
53
+ "special": true
54
+ },
55
+ "6": {
56
+ "content": "<cls>",
57
+ "lstrip": false,
58
+ "normalized": false,
59
+ "rstrip": false,
60
+ "single_word": false,
61
+ "special": true
62
+ },
63
+ "7": {
64
+ "content": "<|system|>",
65
+ "lstrip": false,
66
+ "normalized": false,
67
+ "rstrip": false,
68
+ "single_word": false,
69
+ "special": false
70
+ },
71
+ "8": {
72
+ "content": "<|assistant|>",
73
+ "lstrip": false,
74
+ "normalized": false,
75
+ "rstrip": false,
76
+ "single_word": false,
77
+ "special": false
78
+ },
79
+ "9": {
80
+ "content": "<|user|>",
81
+ "lstrip": false,
82
+ "normalized": false,
83
+ "rstrip": false,
84
+ "single_word": false,
85
+ "special": false
86
+ },
87
+ "10": {
88
+ "content": "<|available_tools|>",
89
+ "lstrip": false,
90
+ "normalized": false,
91
+ "rstrip": false,
92
+ "single_word": false,
93
+ "special": false
94
+ },
95
+ "11": {
96
+ "content": "<|tool_calls|>",
97
+ "lstrip": false,
98
+ "normalized": false,
99
+ "rstrip": false,
100
+ "single_word": false,
101
+ "special": false
102
+ },
103
+ "12": {
104
+ "content": "<|tool_results|>",
105
+ "lstrip": false,
106
+ "normalized": false,
107
+ "rstrip": false,
108
+ "single_word": false,
109
+ "special": false
110
+ },
111
+ "13": {
112
+ "content": "<|code|>",
113
+ "lstrip": false,
114
+ "normalized": false,
115
+ "rstrip": false,
116
+ "single_word": false,
117
+ "special": false
118
+ },
119
+ "14": {
120
+ "content": "<|file|>",
121
+ "lstrip": false,
122
+ "normalized": false,
123
+ "rstrip": false,
124
+ "single_word": false,
125
+ "special": false
126
+ },
127
+ "102397": {
128
+ "content": "<|prefix|>",
129
+ "lstrip": false,
130
+ "normalized": false,
131
+ "rstrip": false,
132
+ "single_word": false,
133
+ "special": false
134
+ },
135
+ "102398": {
136
+ "content": "<|suffix|>",
137
+ "lstrip": false,
138
+ "normalized": false,
139
+ "rstrip": false,
140
+ "single_word": false,
141
+ "special": false
142
+ },
143
+ "102399": {
144
+ "content": "<|middle|>",
145
+ "lstrip": false,
146
+ "normalized": false,
147
+ "rstrip": false,
148
+ "single_word": false,
149
+ "special": false
150
  }
151
  },
152
+ "bos_token": "<s>",
153
+ "clean_up_tokenization_spaces": false,
154
+ "cls_token": "<cls>",
155
+ "do_lower_case": false,
156
+ "eos_token": "</s>",
157
+ "extra_ids": 0,
158
+ "keep_accents": true,
159
+ "legacy": false,
160
+ "mask_token": "<mask>",
161
+ "model_max_length": 8192,
162
+ "pad_token": "<pad>",
163
+ "padding_side": "left",
164
+ "sep_token": "<sep>",
165
+ "sp_model_kwargs": {},
166
+ "spaces_between_special_tokens": false,
167
+ "tokenizer_class": "LlamaTokenizer",
168
+ "unk_token": "<unk>",
169
+ "use_default_system_prompt": false
 
 
170
  }
vocabulary.json CHANGED
The diff for this file is too large to render. See raw diff