rajammanabrolu
/

gpt-4-chat-debug

Model card Files Files and versions

rajammanabrolu commited on Nov 30, 2023

Commit

fb8a662

·

1 Parent(s): e30d950

Update tiktoken.py

Files changed (1) hide show

tiktoken.py +2 -1

tiktoken.py CHANGED Viewed

@@ -215,7 +215,8 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             '{% if (add_generation_prompt == true) %}'
             "{{'<|im_start|>' + 'assistant' + '\n'}}"
             "{% else %}"
-            "{{ eos_token }}")
         template = template.replace(
             'USE_DEFAULT_PROMPT',
             'true' if self.use_default_system_prompt else 'false')

             '{% if (add_generation_prompt == true) %}'
             "{{'<|im_start|>' + 'assistant' + '\n'}}"
             "{% else %}"
+            "{{ eos_token }}"
+            "{% endif %}")
         template = template.replace(
             'USE_DEFAULT_PROMPT',
             'true' if self.use_default_system_prompt else 'false')