Hugging Face's logo

andro-flock
/
wiki_llama_tokenizer

Transformers
Model card Files
xet
Community
andro-flock commited on
Commit
1b637fe
·
verified ·
1 Parent(s): 271abde

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +19 -19
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -4,7 +4,7 @@
4
  "add_prefix_space": null,
5
  "added_tokens_decoder": {
6
  "0": {
7
- "content": "<|endoftext|>",
8
  "lstrip": false,
9
  "normalized": false,
10
  "rstrip": false,
@@ -12,7 +12,7 @@
12
  "special": true
13
  },
14
  "1": {
15
- "content": "<|im_start|>",
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
@@ -20,7 +20,7 @@
20
  "special": true
21
  },
22
  "2": {
23
- "content": "<|im_end|>",
24
  "lstrip": false,
25
  "normalized": false,
26
  "rstrip": false,
@@ -28,7 +28,7 @@
28
  "special": true
29
  },
30
  "3": {
31
- "content": "<repo_name>",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
@@ -36,7 +36,7 @@
36
  "special": true
37
  },
38
  "4": {
39
- "content": "<reponame>",
40
  "lstrip": false,
41
  "normalized": false,
42
  "rstrip": false,
@@ -44,7 +44,7 @@
44
  "special": true
45
  },
46
  "5": {
47
- "content": "<file_sep>",
48
  "lstrip": false,
49
  "normalized": false,
50
  "rstrip": false,
@@ -52,7 +52,7 @@
52
  "special": true
53
  },
54
  "6": {
55
- "content": "<filename>",
56
  "lstrip": false,
57
  "normalized": false,
58
  "rstrip": false,
@@ -60,7 +60,7 @@
60
  "special": true
61
  },
62
  "7": {
63
- "content": "<gh_stars>",
64
  "lstrip": false,
65
  "normalized": false,
66
  "rstrip": false,
@@ -68,7 +68,7 @@
68
  "special": true
69
  },
70
  "8": {
71
- "content": "<issue_start>",
72
  "lstrip": false,
73
  "normalized": false,
74
  "rstrip": false,
@@ -76,7 +76,7 @@
76
  "special": true
77
  },
78
  "9": {
79
- "content": "<issue_comment>",
80
  "lstrip": false,
81
  "normalized": false,
82
  "rstrip": false,
@@ -84,7 +84,7 @@
84
  "special": true
85
  },
86
  "10": {
87
- "content": "<issue_closed>",
88
  "lstrip": false,
89
  "normalized": false,
90
  "rstrip": false,
@@ -92,7 +92,7 @@
92
  "special": true
93
  },
94
  "11": {
95
- "content": "<jupyter_start>",
96
  "lstrip": false,
97
  "normalized": false,
98
  "rstrip": false,
@@ -100,7 +100,7 @@
100
  "special": true
101
  },
102
  "12": {
103
- "content": "<jupyter_text>",
104
  "lstrip": false,
105
  "normalized": false,
106
  "rstrip": false,
@@ -108,7 +108,7 @@
108
  "special": true
109
  },
110
  "13": {
111
- "content": "<jupyter_code>",
112
  "lstrip": false,
113
  "normalized": false,
114
  "rstrip": false,
@@ -116,7 +116,7 @@
116
  "special": true
117
  },
118
  "14": {
119
- "content": "<jupyter_output>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
@@ -124,7 +124,7 @@
124
  "special": true
125
  },
126
  "15": {
127
- "content": "<jupyter_script>",
128
  "lstrip": false,
129
  "normalized": false,
130
  "rstrip": false,
@@ -132,15 +132,15 @@
132
  "special": true
133
  },
134
  "16": {
135
- "content": "<empty_output>",
136
  "lstrip": false,
137
  "normalized": false,
138
  "rstrip": false,
139
  "single_word": false,
140
  "special": true
141
  },
142
- "15000": {
143
- "content": "<unk>",
144
  "lstrip": false,
145
  "normalized": false,
146
  "rstrip": false,
 
4
  "add_prefix_space": null,
5
  "added_tokens_decoder": {
6
  "0": {
7
+ "content": "<unk>",
8
  "lstrip": false,
9
  "normalized": false,
10
  "rstrip": false,
 
12
  "special": true
13
  },
14
  "1": {
15
+ "content": "<|endoftext|>",
16
  "lstrip": false,
17
  "normalized": false,
18
  "rstrip": false,
 
20
  "special": true
21
  },
22
  "2": {
23
+ "content": "<|im_start|>",
24
  "lstrip": false,
25
  "normalized": false,
26
  "rstrip": false,
 
28
  "special": true
29
  },
30
  "3": {
31
+ "content": "<|im_end|>",
32
  "lstrip": false,
33
  "normalized": false,
34
  "rstrip": false,
 
36
  "special": true
37
  },
38
  "4": {
39
+ "content": "<repo_name>",
40
  "lstrip": false,
41
  "normalized": false,
42
  "rstrip": false,
 
44
  "special": true
45
  },
46
  "5": {
47
+ "content": "<reponame>",
48
  "lstrip": false,
49
  "normalized": false,
50
  "rstrip": false,
 
52
  "special": true
53
  },
54
  "6": {
55
+ "content": "<file_sep>",
56
  "lstrip": false,
57
  "normalized": false,
58
  "rstrip": false,
 
60
  "special": true
61
  },
62
  "7": {
63
+ "content": "<filename>",
64
  "lstrip": false,
65
  "normalized": false,
66
  "rstrip": false,
 
68
  "special": true
69
  },
70
  "8": {
71
+ "content": "<gh_stars>",
72
  "lstrip": false,
73
  "normalized": false,
74
  "rstrip": false,
 
76
  "special": true
77
  },
78
  "9": {
79
+ "content": "<issue_start>",
80
  "lstrip": false,
81
  "normalized": false,
82
  "rstrip": false,
 
84
  "special": true
85
  },
86
  "10": {
87
+ "content": "<issue_comment>",
88
  "lstrip": false,
89
  "normalized": false,
90
  "rstrip": false,
 
92
  "special": true
93
  },
94
  "11": {
95
+ "content": "<issue_closed>",
96
  "lstrip": false,
97
  "normalized": false,
98
  "rstrip": false,
 
100
  "special": true
101
  },
102
  "12": {
103
+ "content": "<jupyter_start>",
104
  "lstrip": false,
105
  "normalized": false,
106
  "rstrip": false,
 
108
  "special": true
109
  },
110
  "13": {
111
+ "content": "<jupyter_text>",
112
  "lstrip": false,
113
  "normalized": false,
114
  "rstrip": false,
 
116
  "special": true
117
  },
118
  "14": {
119
+ "content": "<jupyter_code>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
 
124
  "special": true
125
  },
126
  "15": {
127
+ "content": "<jupyter_output>",
128
  "lstrip": false,
129
  "normalized": false,
130
  "rstrip": false,
 
132
  "special": true
133
  },
134
  "16": {
135
+ "content": "<jupyter_script>",
136
  "lstrip": false,
137
  "normalized": false,
138
  "rstrip": false,
139
  "single_word": false,
140
  "special": true
141
  },
142
+ "17": {
143
+ "content": "<empty_output>",
144
  "lstrip": false,
145
  "normalized": false,
146
  "rstrip": false,