File size: 2,586 Bytes
42621a1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | {
"add_bos_token": false,
"add_eos_token": false,
"bos_token": "<|begin_of_text|>",
"eos_token": "<|end_of_text|>",
"pad_token": "<|padding|>",
"unk_token": "<|unknown|>",
"model_max_length": 2048,
"clean_up_tokenization_spaces": false,
"tokenizer_class": "PreTrainedTokenizerFast",
"chat_template": "{%- for message in messages -%}\n{%- if loop.first -%}{{- bos_token -}}{%- endif -%}\n{{- '<|im_start|>' + message['role'] + '\\n' -}}\n{%- if message['role'] == 'tool' -%}{{- '<|tool_response|>\\n' -}}{%- endif -%}\n{%- if message['content'] is string -%}\n{{- message['content'] -}}\n{%- elif message['content'] is iterable -%}\n{%- for item in message['content'] -%}\n{%- if item['type'] == 'text' -%}{{- item['text'] -}}{%- endif -%}\n{%- endfor -%}\n{%- endif -%}\n{%- if message['tool_calls'] is defined and message['tool_calls'] -%}\n{{- '\\n<|tool_call|>\\n' + (message['tool_calls'] | tojson) -}}\n{%- endif -%}\n{{- '<|im_end|>\\n' -}}\n{%- endfor -%}\n{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|tool_call|>",
"<|tool_response|>",
"<|think_start|>",
"<|think_end|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|fim_pad|>",
"<|reserved_000|>",
"<|reserved_001|>",
"<|reserved_002|>",
"<|reserved_003|>",
"<|reserved_004|>",
"<|reserved_005|>",
"<|reserved_006|>",
"<|reserved_007|>",
"<|reserved_008|>",
"<|reserved_009|>",
"<|reserved_010|>",
"<|reserved_011|>",
"<|reserved_012|>",
"<|reserved_013|>",
"<|reserved_014|>",
"<|reserved_015|>",
"<|reserved_016|>",
"<|reserved_017|>",
"<|reserved_018|>",
"<|reserved_019|>",
"<|reserved_020|>",
"<|reserved_021|>",
"<|reserved_022|>",
"<|reserved_023|>",
"<|reserved_024|>",
"<|reserved_025|>",
"<|reserved_026|>",
"<|reserved_027|>",
"<|reserved_028|>",
"<|reserved_029|>",
"<|reserved_030|>",
"<|reserved_031|>",
"<|reserved_032|>",
"<|reserved_033|>",
"<|reserved_034|>",
"<|reserved_035|>",
"<|reserved_036|>",
"<|reserved_037|>",
"<|reserved_038|>",
"<|reserved_039|>",
"<|reserved_040|>",
"<|reserved_041|>",
"<|reserved_042|>",
"<|reserved_043|>",
"<|reserved_044|>",
"<|reserved_045|>",
"<|reserved_046|>",
"<|reserved_047|>",
"<|reserved_048|>",
"<|reserved_049|>"
]
}
|