rajammanabrolu
/

gpt-4-chat-debug

Model card Files Files and versions

rajammanabrolu commited on Nov 30, 2023

Commit

203d323

·

1 Parent(s): 02b52ab

Update tiktoken.py

Files changed (1) hide show

tiktoken.py +1 -1

tiktoken.py CHANGED Viewed

@@ -207,7 +207,7 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             "{% if loop.index0 == 0 and system_message != false %}"  # Embed system message in first message
             "{{ '<|im_start|>system\n' + system_message.strip() }}"
             "{% else %}"
-            "{{ '\n' + <|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}"
             "{% endif %}"
             '{% if (add_generation_prompt == true) %}'
             "{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}"

             "{% if loop.index0 == 0 and system_message != false %}"  # Embed system message in first message
             "{{ '<|im_start|>system\n' + system_message.strip() }}"
             "{% else %}"
+            "{{ '\n' + '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}"
             "{% endif %}"
             '{% if (add_generation_prompt == true) %}'
             "{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}"