Commit ·
3cb3664
1
Parent(s): fb8a662
Update tiktoken.py
Browse files- tiktoken.py +2 -4
tiktoken.py
CHANGED
|
@@ -204,9 +204,6 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
|
|
| 204 |
"{% set system_message = false %}"
|
| 205 |
"{% endif %}"
|
| 206 |
"{% for message in loop_messages %}" # Loop over all non-system messages
|
| 207 |
-
"{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}"
|
| 208 |
-
"{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}"
|
| 209 |
-
"{% endif %}"
|
| 210 |
"{% if loop.index0 == 0 and system_message != false %}" # Embed system message in first message
|
| 211 |
"{{ '<|im_start|>system\n' + system_message.strip() + '\n'}}"
|
| 212 |
"{% else %}"
|
|
@@ -216,7 +213,8 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
|
|
| 216 |
"{{'<|im_start|>' + 'assistant' + '\n'}}"
|
| 217 |
"{% else %}"
|
| 218 |
"{{ eos_token }}"
|
| 219 |
-
"{% endif %}"
|
|
|
|
| 220 |
template = template.replace(
|
| 221 |
'USE_DEFAULT_PROMPT',
|
| 222 |
'true' if self.use_default_system_prompt else 'false')
|
|
|
|
| 204 |
"{% set system_message = false %}"
|
| 205 |
"{% endif %}"
|
| 206 |
"{% for message in loop_messages %}" # Loop over all non-system messages
|
|
|
|
|
|
|
|
|
|
| 207 |
"{% if loop.index0 == 0 and system_message != false %}" # Embed system message in first message
|
| 208 |
"{{ '<|im_start|>system\n' + system_message.strip() + '\n'}}"
|
| 209 |
"{% else %}"
|
|
|
|
| 213 |
"{{'<|im_start|>' + 'assistant' + '\n'}}"
|
| 214 |
"{% else %}"
|
| 215 |
"{{ eos_token }}"
|
| 216 |
+
"{% endif %}"
|
| 217 |
+
'{% endfor %}')
|
| 218 |
template = template.replace(
|
| 219 |
'USE_DEFAULT_PROMPT',
|
| 220 |
'true' if self.use_default_system_prompt else 'false')
|