neshkatrapati commited on
Commit
e0ef6c3
·
verified ·
1 Parent(s): 4f89dcb

Fix tokenizer.json: update to SFT chat model config

Browse files
Files changed (1) hide show
  1. tokenizer.json +41 -1
tokenizer.json CHANGED
@@ -38,6 +38,42 @@
38
  "rstrip": false,
39
  "normalized": false,
40
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  }
42
  ],
43
  "normalizer": null,
@@ -86166,7 +86202,11 @@
86166
  "౿": 86067,
86167
  "౿@@": 86068,
86168
  "—": 86069,
86169
- "—@@": 86070
 
 
 
 
86170
  },
86171
  "unk_token": "<unk>"
86172
  }
 
38
  "rstrip": false,
39
  "normalized": false,
40
  "special": true
41
+ },
42
+ {
43
+ "id": 86071,
44
+ "content": "<|system|>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 86072,
53
+ "content": "<|user|>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ },
60
+ {
61
+ "id": 86073,
62
+ "content": "<|assistant|>",
63
+ "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
+ },
69
+ {
70
+ "id": 86074,
71
+ "content": "<|end|>",
72
+ "single_word": false,
73
+ "lstrip": false,
74
+ "rstrip": false,
75
+ "normalized": false,
76
+ "special": true
77
  }
78
  ],
79
  "normalizer": null,
 
86202
  "౿": 86067,
86203
  "౿@@": 86068,
86204
  "—": 86069,
86205
+ "—@@": 86070,
86206
+ "<|system|>": 86071,
86207
+ "<|user|>": 86072,
86208
+ "<|assistant|>": 86073,
86209
+ "<|end|>": 86074
86210
  },
86211
  "unk_token": "<unk>"
86212
  }