{ "add_prefix_space": false, "added_tokens_decoder": { "5809": { "content": "�", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "100256": { "content": "<|dummy_0|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100257": { "content": "<|endoftext|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100258": { "content": "<|fim_prefix|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100259": { "content": "<|fim_middle|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100260": { "content": "<|fim_suffix|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100261": { "content": "<|dummy_1|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100262": { "content": "<|dummy_2|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100263": { "content": "<|dummy_3|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100264": { "content": "<|im_start|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100265": { "content": "<|im_end|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100266": { "content": "<|im_sep|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100267": { "content": "<|dummy_4|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100268": { "content": "<|dummy_5|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100269": { "content": "<|dummy_6|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100270": { "content": "<|dummy_7|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100271": { "content": "<|dummy_8|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100272": { "content": "<|dummy_9|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100273": { "content": "<|dummy_10|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100274": { "content": "<|dummy_11|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100275": { "content": "<|dummy_12|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100276": { "content": "<|endofprompt|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100277": { "content": "<|dummy_13|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100278": { "content": "<|dummy_14|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100279": { "content": "<|dummy_15|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100280": { "content": "<|dummy_16|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100281": { "content": "<|dummy_17|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100282": { "content": "<|dummy_18|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100283": { "content": "<|dummy_19|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100284": { "content": "<|dummy_20|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100285": { "content": "<|dummy_21|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100286": { "content": "<|dummy_22|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100287": { "content": "<|dummy_23|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100288": { "content": "<|dummy_24|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100289": { "content": "<|dummy_25|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100290": { "content": "<|dummy_26|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100291": { "content": "<|dummy_27|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100292": { "content": "<|dummy_28|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100293": { "content": "<|dummy_29|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100294": { "content": "<|dummy_30|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100295": { "content": "<|dummy_31|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100296": { "content": "<|dummy_32|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100297": { "content": "<|dummy_33|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100298": { "content": "<|dummy_34|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100299": { "content": "<|dummy_35|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100300": { "content": "<|dummy_36|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100301": { "content": "<|dummy_37|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100302": { "content": "<|dummy_38|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100303": { "content": "<|dummy_39|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100304": { "content": "<|dummy_40|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100305": { "content": "<|dummy_41|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100306": { "content": "<|dummy_42|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100307": { "content": "<|dummy_43|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100308": { "content": "<|dummy_44|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100309": { "content": "<|dummy_45|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100310": { "content": "<|dummy_46|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100311": { "content": "<|dummy_47|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100312": { "content": "<|dummy_48|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100313": { "content": "<|dummy_49|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100314": { "content": "<|dummy_50|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100315": { "content": "<|dummy_51|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100316": { "content": "<|dummy_52|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100317": { "content": "<|dummy_53|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100318": { "content": "<|dummy_54|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100319": { "content": "<|dummy_55|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100320": { "content": "<|dummy_56|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100321": { "content": "<|dummy_57|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100322": { "content": "<|dummy_58|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100323": { "content": "<|dummy_59|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100324": { "content": "<|dummy_60|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100325": { "content": "<|dummy_61|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100326": { "content": "<|dummy_62|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100327": { "content": "<|dummy_63|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100328": { "content": "<|dummy_64|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100329": { "content": "<|dummy_65|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100330": { "content": "<|dummy_66|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100331": { "content": "<|dummy_67|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100332": { "content": "<|dummy_68|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100333": { "content": "<|dummy_69|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100334": { "content": "<|dummy_70|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100335": { "content": "<|dummy_71|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100336": { "content": "<|dummy_72|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100337": { "content": "<|dummy_73|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100338": { "content": "<|dummy_74|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100339": { "content": "<|dummy_75|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100340": { "content": "<|dummy_76|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100341": { "content": "<|dummy_77|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100342": { "content": "<|dummy_78|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100343": { "content": "<|dummy_79|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100344": { "content": "<|dummy_80|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100345": { "content": "<|dummy_81|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100346": { "content": "<|dummy_82|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100347": { "content": "<|dummy_83|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100348": { "content": "<|dummy_84|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100349": { "content": "<|dummy_85|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100350": { "content": "<|dummy_86|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100351": { "content": "<|dummy_87|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true } }, "bos_token": "<|endoftext|>", "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 July 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\n\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\n\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\n\" }}\n{{- \"Today Date: \" + date_string + \"\n\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\n\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\n\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content'] %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\n\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\n\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\n\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\n\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}\n{%- endif %}\n", "clean_up_tokenization_spaces": false, "eos_token": "<|im_end|>", "extra_special_tokens": {}, "model_max_length": 16384, "pad_token": "<|dummy_87|>", "padding_side": "right", "tokenizer_class": "GPT2Tokenizer", "unk_token": "�" }