rajammanabrolu commited on
Commit
fe3e2e7
·
1 Parent(s): ebaaf48

Update tiktoken.py

Browse files
Files changed (1) hide show
  1. tiktoken.py +2 -7
tiktoken.py CHANGED
@@ -179,17 +179,12 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
179
  '{% endif %}'
180
  '{% for message in messages %}'
181
  "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
182
- '{% if ADD_GENERATION_PROMPT == true %}'
183
- "{{ '<|im_start|>' + 'assistant' + '\n' }}"
184
  '{% endfor %}')
185
  template = template.replace(
186
  'USE_DEFAULT_PROMPT',
187
  'true' if self.use_default_system_prompt else 'false')
188
- template = template.replace(
189
- 'ADD_GENERATION_PROMPT',
190
- 'true' if self.add_generation_prompt else 'false'
191
- )
192
- print("GENERATION PROMPT", self.add_generation_prompt)
193
  template = template.replace('DEFAULT_SYSTEM_PROMPT',
194
  DEFAULT_SYSTEM_PROMPT)
195
  return template
 
179
  '{% endif %}'
180
  '{% for message in messages %}'
181
  "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
182
+ '{% if (add_generation_prompt == true) %}'
183
+ "{{'<|im_start|>' + 'assistant' + '\n'}}"
184
  '{% endfor %}')
185
  template = template.replace(
186
  'USE_DEFAULT_PROMPT',
187
  'true' if self.use_default_system_prompt else 'false')
 
 
 
 
 
188
  template = template.replace('DEFAULT_SYSTEM_PROMPT',
189
  DEFAULT_SYSTEM_PROMPT)
190
  return template