Commit
·
fe3e2e7
1
Parent(s):
ebaaf48
Update tiktoken.py
Browse files- tiktoken.py +2 -7
tiktoken.py
CHANGED
|
@@ -179,17 +179,12 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
|
|
| 179 |
'{% endif %}'
|
| 180 |
'{% for message in messages %}'
|
| 181 |
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
|
| 182 |
-
'{% if
|
| 183 |
-
"{{
|
| 184 |
'{% endfor %}')
|
| 185 |
template = template.replace(
|
| 186 |
'USE_DEFAULT_PROMPT',
|
| 187 |
'true' if self.use_default_system_prompt else 'false')
|
| 188 |
-
template = template.replace(
|
| 189 |
-
'ADD_GENERATION_PROMPT',
|
| 190 |
-
'true' if self.add_generation_prompt else 'false'
|
| 191 |
-
)
|
| 192 |
-
print("GENERATION PROMPT", self.add_generation_prompt)
|
| 193 |
template = template.replace('DEFAULT_SYSTEM_PROMPT',
|
| 194 |
DEFAULT_SYSTEM_PROMPT)
|
| 195 |
return template
|
|
|
|
| 179 |
'{% endif %}'
|
| 180 |
'{% for message in messages %}'
|
| 181 |
"{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
|
| 182 |
+
'{% if (add_generation_prompt == true) %}'
|
| 183 |
+
"{{'<|im_start|>' + 'assistant' + '\n'}}"
|
| 184 |
'{% endfor %}')
|
| 185 |
template = template.replace(
|
| 186 |
'USE_DEFAULT_PROMPT',
|
| 187 |
'true' if self.use_default_system_prompt else 'false')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
template = template.replace('DEFAULT_SYSTEM_PROMPT',
|
| 189 |
DEFAULT_SYSTEM_PROMPT)
|
| 190 |
return template
|