Hugging Face's logo

devngho
/
phi3-jamo-ko-tokenizer

Transformers
Korean
Model card Files
xet
Community
devngho commited on
Commit
2f52aac
·
verified ·
1 Parent(s): 6bb6fd6

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +11 -11
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "56689": {
28
  "content": "<|endoftext|>",
29
  "lstrip": false,
30
  "normalized": false,
@@ -32,7 +32,7 @@
32
  "single_word": false,
33
  "special": true
34
  },
35
- "56690": {
36
  "content": "<|assistant|>",
37
  "lstrip": false,
38
  "normalized": false,
@@ -40,7 +40,7 @@
40
  "single_word": false,
41
  "special": true
42
  },
43
- "56691": {
44
  "content": "<|placeholder1|>",
45
  "lstrip": false,
46
  "normalized": false,
@@ -48,7 +48,7 @@
48
  "single_word": false,
49
  "special": true
50
  },
51
- "56692": {
52
  "content": "<|placeholder2|>",
53
  "lstrip": false,
54
  "normalized": false,
@@ -56,7 +56,7 @@
56
  "single_word": false,
57
  "special": true
58
  },
59
- "56693": {
60
  "content": "<|placeholder3|>",
61
  "lstrip": false,
62
  "normalized": false,
@@ -64,7 +64,7 @@
64
  "single_word": false,
65
  "special": true
66
  },
67
- "56694": {
68
  "content": "<|placeholder4|>",
69
  "lstrip": false,
70
  "normalized": false,
@@ -72,7 +72,7 @@
72
  "single_word": false,
73
  "special": true
74
  },
75
- "56695": {
76
  "content": "<|system|>",
77
  "lstrip": false,
78
  "normalized": false,
@@ -80,7 +80,7 @@
80
  "single_word": false,
81
  "special": true
82
  },
83
- "56696": {
84
  "content": "<|end|>",
85
  "lstrip": false,
86
  "normalized": false,
@@ -88,7 +88,7 @@
88
  "single_word": false,
89
  "special": true
90
  },
91
- "56697": {
92
  "content": "<|placeholder5|>",
93
  "lstrip": false,
94
  "normalized": false,
@@ -96,7 +96,7 @@
96
  "single_word": false,
97
  "special": true
98
  },
99
- "56698": {
100
  "content": "<|placeholder6|>",
101
  "lstrip": false,
102
  "normalized": false,
@@ -104,7 +104,7 @@
104
  "single_word": false,
105
  "special": true
106
  },
107
- "56699": {
108
  "content": "<|user|>",
109
  "lstrip": false,
110
  "normalized": false,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "32000": {
28
  "content": "<|endoftext|>",
29
  "lstrip": false,
30
  "normalized": false,
 
32
  "single_word": false,
33
  "special": true
34
  },
35
+ "32001": {
36
  "content": "<|assistant|>",
37
  "lstrip": false,
38
  "normalized": false,
 
40
  "single_word": false,
41
  "special": true
42
  },
43
+ "32002": {
44
  "content": "<|placeholder1|>",
45
  "lstrip": false,
46
  "normalized": false,
 
48
  "single_word": false,
49
  "special": true
50
  },
51
+ "32003": {
52
  "content": "<|placeholder2|>",
53
  "lstrip": false,
54
  "normalized": false,
 
56
  "single_word": false,
57
  "special": true
58
  },
59
+ "32004": {
60
  "content": "<|placeholder3|>",
61
  "lstrip": false,
62
  "normalized": false,
 
64
  "single_word": false,
65
  "special": true
66
  },
67
+ "32005": {
68
  "content": "<|placeholder4|>",
69
  "lstrip": false,
70
  "normalized": false,
 
72
  "single_word": false,
73
  "special": true
74
  },
75
+ "32006": {
76
  "content": "<|system|>",
77
  "lstrip": false,
78
  "normalized": false,
 
80
  "single_word": false,
81
  "special": true
82
  },
83
+ "32007": {
84
  "content": "<|end|>",
85
  "lstrip": false,
86
  "normalized": false,
 
88
  "single_word": false,
89
  "special": true
90
  },
91
+ "32008": {
92
  "content": "<|placeholder5|>",
93
  "lstrip": false,
94
  "normalized": false,
 
96
  "single_word": false,
97
  "special": true
98
  },
99
+ "32009": {
100
  "content": "<|placeholder6|>",
101
  "lstrip": false,
102
  "normalized": false,
 
104
  "single_word": false,
105
  "special": true
106
  },
107
+ "32010": {
108
  "content": "<|user|>",
109
  "lstrip": false,
110
  "normalized": false,