Add chat template

#1
by emilss - opened
Files changed (1) hide show
  1. tokenizer_config.json +23 -1
tokenizer_config.json CHANGED
@@ -8002,13 +8002,35 @@
8002
  "rstrip": false,
8003
  "single_word": false,
8004
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8005
  }
8006
  },
8007
  "bos_token": "<s>",
 
8008
  "clean_up_tokenization_spaces": false,
8009
  "eos_token": "</s>",
8010
  "extra_special_tokens": {},
 
 
 
 
8011
  "model_max_length": 1000000000000000019884624838656,
 
8012
  "tokenizer_class": "PreTrainedTokenizerFast",
8013
  "unk_token": "<unk>"
8014
- }
 
8002
  "rstrip": false,
8003
  "single_word": false,
8004
  "special": true
8005
+ },
8006
+ "131072": {
8007
+ "content": "<|im_start|>",
8008
+ "lstrip": false,
8009
+ "normalized": false,
8010
+ "rstrip": false,
8011
+ "single_word": false,
8012
+ "special": false
8013
+ },
8014
+ "131073": {
8015
+ "content": "<|im_end|>",
8016
+ "lstrip": false,
8017
+ "normalized": false,
8018
+ "rstrip": false,
8019
+ "single_word": false,
8020
+ "special": false
8021
  }
8022
  },
8023
  "bos_token": "<s>",
8024
+ "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
8025
  "clean_up_tokenization_spaces": false,
8026
  "eos_token": "</s>",
8027
  "extra_special_tokens": {},
8028
+ "model_input_names": [
8029
+ "input_ids",
8030
+ "attention_mask"
8031
+ ],
8032
  "model_max_length": 1000000000000000019884624838656,
8033
+ "pad_token": "<pad>",
8034
  "tokenizer_class": "PreTrainedTokenizerFast",
8035
  "unk_token": "<unk>"
8036
+ }