Hugging Face's logo

styalai
/
XT-test-0.1

Transformers
Safetensors
model_hub_mixin
pytorch_model_hub_mixin
Model card Files
xet
Community
styalai commited on
Commit
4461d51
·
verified ·
1 Parent(s): 9c24071

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +14 -14
  2. tokenizer_config.json +14 -14
tokenizer.json CHANGED
@@ -27,17 +27,17 @@
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
- "normalized": false,
31
- "special": true
32
  },
33
  {
34
  "id": 3,
35
  "content": "<|assistant|>",
36
  "single_word": false,
37
  "lstrip": false,
38
- "rstrip": true,
39
- "normalized": false,
40
- "special": true
41
  },
42
  {
43
  "id": 4,
@@ -80,18 +80,18 @@
80
  "content": "<|system|>",
81
  "single_word": false,
82
  "lstrip": false,
83
- "rstrip": true,
84
- "normalized": false,
85
- "special": true
86
  },
87
  {
88
  "id": 9,
89
  "content": "<|end|>",
90
  "single_word": false,
91
  "lstrip": false,
92
- "rstrip": true,
93
- "normalized": false,
94
- "special": true
95
  },
96
  {
97
  "id": 10,
@@ -116,9 +116,9 @@
116
  "content": "<|user|>",
117
  "single_word": false,
118
  "lstrip": false,
119
- "rstrip": true,
120
- "normalized": false,
121
- "special": true
122
  }
123
  ],
124
  "normalizer": {
 
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
+ "normalized": true,
31
+ "special": false
32
  },
33
  {
34
  "id": 3,
35
  "content": "<|assistant|>",
36
  "single_word": false,
37
  "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": true,
40
+ "special": false
41
  },
42
  {
43
  "id": 4,
 
80
  "content": "<|system|>",
81
  "single_word": false,
82
  "lstrip": false,
83
+ "rstrip": false,
84
+ "normalized": true,
85
+ "special": false
86
  },
87
  {
88
  "id": 9,
89
  "content": "<|end|>",
90
  "single_word": false,
91
  "lstrip": false,
92
+ "rstrip": false,
93
+ "normalized": true,
94
+ "special": false
95
  },
96
  {
97
  "id": 10,
 
116
  "content": "<|user|>",
117
  "single_word": false,
118
  "lstrip": false,
119
+ "rstrip": false,
120
+ "normalized": true,
121
+ "special": false
122
  }
123
  ],
124
  "normalizer": {
tokenizer_config.json CHANGED
@@ -21,18 +21,18 @@
21
  "2": {
22
  "content": "<|endoftext|>",
23
  "lstrip": false,
24
- "normalized": false,
25
  "rstrip": false,
26
  "single_word": false,
27
- "special": true
28
  },
29
  "3": {
30
  "content": "<|assistant|>",
31
  "lstrip": false,
32
- "normalized": false,
33
- "rstrip": true,
34
  "single_word": false,
35
- "special": true
36
  },
37
  "4": {
38
  "content": "<|placeholder1|>",
@@ -69,18 +69,18 @@
69
  "8": {
70
  "content": "<|system|>",
71
  "lstrip": false,
72
- "normalized": false,
73
- "rstrip": true,
74
  "single_word": false,
75
- "special": true
76
  },
77
  "9": {
78
  "content": "<|end|>",
79
  "lstrip": false,
80
- "normalized": false,
81
- "rstrip": true,
82
  "single_word": false,
83
- "special": true
84
  },
85
  "10": {
86
  "content": "<|placeholder5|>",
@@ -101,10 +101,10 @@
101
  "12": {
102
  "content": "<|user|>",
103
  "lstrip": false,
104
- "normalized": false,
105
- "rstrip": true,
106
  "single_word": false,
107
- "special": true
108
  }
109
  },
110
  "bos_token": "<s>",
 
21
  "2": {
22
  "content": "<|endoftext|>",
23
  "lstrip": false,
24
+ "normalized": true,
25
  "rstrip": false,
26
  "single_word": false,
27
+ "special": false
28
  },
29
  "3": {
30
  "content": "<|assistant|>",
31
  "lstrip": false,
32
+ "normalized": true,
33
+ "rstrip": false,
34
  "single_word": false,
35
+ "special": false
36
  },
37
  "4": {
38
  "content": "<|placeholder1|>",
 
69
  "8": {
70
  "content": "<|system|>",
71
  "lstrip": false,
72
+ "normalized": true,
73
+ "rstrip": false,
74
  "single_word": false,
75
+ "special": false
76
  },
77
  "9": {
78
  "content": "<|end|>",
79
  "lstrip": false,
80
+ "normalized": true,
81
+ "rstrip": false,
82
  "single_word": false,
83
+ "special": false
84
  },
85
  "10": {
86
  "content": "<|placeholder5|>",
 
101
  "12": {
102
  "content": "<|user|>",
103
  "lstrip": false,
104
+ "normalized": true,
105
+ "rstrip": false,
106
  "single_word": false,
107
+ "special": false
108
  }
109
  },
110
  "bos_token": "<s>",