DanielHesslow commited on
Commit
0e12b9d
·
verified ·
1 Parent(s): 3763f71

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +9 -0
tokenizer.json CHANGED
@@ -12,6 +12,15 @@
12
  "normalized": false,
13
  "special": true
14
  },
 
 
 
 
 
 
 
 
 
15
  {
16
  "id": 257,
17
  "content": "<end_of_turn>",
 
12
  "normalized": false,
13
  "special": true
14
  },
15
+ {
16
+ "id": 0,
17
+ "content": "\\0",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
  {
25
  "id": 257,
26
  "content": "<end_of_turn>",