Update tokenizer.json

#6
by iamPi - opened
Files changed (1) hide show
  1. tokenizer.json +1 -4
tokenizer.json CHANGED
@@ -80,10 +80,7 @@
80
  "9": 9,
81
  "Ċ": 10,
82
  "user": 11,
83
- "assistant": 12,
84
- "<|endoftext|>": 13,
85
- "<|im_start|>": 14,
86
- "<|im_end|>": 15
87
  },
88
  "unk_token": "<|endoftext|>"
89
  }
 
80
  "9": 9,
81
  "Ċ": 10,
82
  "user": 11,
83
+ "assistant": 12
 
 
 
84
  },
85
  "unk_token": "<|endoftext|>"
86
  }