rajammanabrolu
/

gpt-4-chat-debug

Model card Files Files and versions

rajammanabrolu commited on Nov 30, 2023

Commit

e30d950

·

1 Parent(s): a755aa7

Update tiktoken.py

Files changed (1) hide show

tiktoken.py +1 -1

tiktoken.py CHANGED Viewed

@@ -215,7 +215,7 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             '{% if (add_generation_prompt == true) %}'
             "{{'<|im_start|>' + 'assistant' + '\n'}}"
             "{% else %}"
-            "{{ eos_token }}"
         template = template.replace(
             'USE_DEFAULT_PROMPT',
             'true' if self.use_default_system_prompt else 'false')

             '{% if (add_generation_prompt == true) %}'
             "{{'<|im_start|>' + 'assistant' + '\n'}}"
             "{% else %}"
+            "{{ eos_token }}")
         template = template.replace(
             'USE_DEFAULT_PROMPT',
             'true' if self.use_default_system_prompt else 'false')