rajammanabrolu commited on
Commit
3cb3664
·
1 Parent(s): fb8a662

Update tiktoken.py

Browse files
Files changed (1) hide show
  1. tiktoken.py +2 -4
tiktoken.py CHANGED
@@ -204,9 +204,6 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
204
  "{% set system_message = false %}"
205
  "{% endif %}"
206
  "{% for message in loop_messages %}" # Loop over all non-system messages
207
- "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}"
208
- "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}"
209
- "{% endif %}"
210
  "{% if loop.index0 == 0 and system_message != false %}" # Embed system message in first message
211
  "{{ '<|im_start|>system\n' + system_message.strip() + '\n'}}"
212
  "{% else %}"
@@ -216,7 +213,8 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
216
  "{{'<|im_start|>' + 'assistant' + '\n'}}"
217
  "{% else %}"
218
  "{{ eos_token }}"
219
- "{% endif %}")
 
220
  template = template.replace(
221
  'USE_DEFAULT_PROMPT',
222
  'true' if self.use_default_system_prompt else 'false')
 
204
  "{% set system_message = false %}"
205
  "{% endif %}"
206
  "{% for message in loop_messages %}" # Loop over all non-system messages
 
 
 
207
  "{% if loop.index0 == 0 and system_message != false %}" # Embed system message in first message
208
  "{{ '<|im_start|>system\n' + system_message.strip() + '\n'}}"
209
  "{% else %}"
 
213
  "{{'<|im_start|>' + 'assistant' + '\n'}}"
214
  "{% else %}"
215
  "{{ eos_token }}"
216
+ "{% endif %}"
217
+ '{% endfor %}')
218
  template = template.replace(
219
  'USE_DEFAULT_PROMPT',
220
  'true' if self.use_default_system_prompt else 'false')