Upload 4 files
Browse files- chat_template.jinja +31 -0
- special_tokens_map.json +12 -0
- tokenizer.json +0 -0
- tokenizer_config.json +50 -0
chat_template.jinja
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if messages[0].role != 'system' %}
|
| 2 |
+
{{- '<|im_start|>system
|
| 3 |
+
You are a helpful AI assistant powered by Spiking Neural Networks (SNNs), created by Cihan Yalçın. You are an advanced SpikingLLM designed to provide accurate, helpful, and concise responses in English. You utilize biologically-inspired neuron models for energy-efficient processing.<|im_end|>
|
| 4 |
+
' }}
|
| 5 |
+
{%- endif %}
|
| 6 |
+
|
| 7 |
+
{%- for message in messages %}
|
| 8 |
+
{%- if message.content is string %}
|
| 9 |
+
{%- set content = message.content %}
|
| 10 |
+
{%- else %}
|
| 11 |
+
{%- set content = '' %}
|
| 12 |
+
{%- endif %}
|
| 13 |
+
|
| 14 |
+
{%- if message.role == "system" %}
|
| 15 |
+
{{- '<|im_start|>system
|
| 16 |
+
' + content + '<|im_end|>
|
| 17 |
+
' }}
|
| 18 |
+
{%- elif message.role == "user" %}
|
| 19 |
+
{{- '<|im_start|>user
|
| 20 |
+
' + content + '<|im_end|>
|
| 21 |
+
' }}
|
| 22 |
+
{%- elif message.role == "assistant" %}
|
| 23 |
+
{{- '<|im_start|>assistant
|
| 24 |
+
' + content + '<|im_end|>
|
| 25 |
+
' }}
|
| 26 |
+
{%- endif %}
|
| 27 |
+
{%- endfor %}
|
| 28 |
+
{%- if add_generation_prompt %}
|
| 29 |
+
{{- '<|im_start|>assistant
|
| 30 |
+
' }}
|
| 31 |
+
{%- endif %}
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|endoftext|>",
|
| 4 |
+
"<|im_start|>",
|
| 5 |
+
"<|im_end|>",
|
| 6 |
+
"<|pad|>"
|
| 7 |
+
],
|
| 8 |
+
"bos_token": "<|im_start|>",
|
| 9 |
+
"eos_token": "<|endoftext|>",
|
| 10 |
+
"pad_token": "<|pad|>",
|
| 11 |
+
"unk_token": "<|endoftext|>"
|
| 12 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "<|endoftext|>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"1": {
|
| 12 |
+
"content": "<|im_start|>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"2": {
|
| 20 |
+
"content": "<|im_end|>",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": true
|
| 26 |
+
},
|
| 27 |
+
"3": {
|
| 28 |
+
"content": "<|pad|>",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": true
|
| 34 |
+
}
|
| 35 |
+
},
|
| 36 |
+
"additional_special_tokens": [
|
| 37 |
+
"<|endoftext|>",
|
| 38 |
+
"<|im_start|>",
|
| 39 |
+
"<|im_end|>",
|
| 40 |
+
"<|pad|>"
|
| 41 |
+
],
|
| 42 |
+
"bos_token": "<|im_start|>",
|
| 43 |
+
"clean_up_tokenization_spaces": false,
|
| 44 |
+
"eos_token": "<|endoftext|>",
|
| 45 |
+
"extra_special_tokens": {},
|
| 46 |
+
"model_max_length": 1024,
|
| 47 |
+
"pad_token": "<|pad|>",
|
| 48 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 49 |
+
"unk_token": "<|endoftext|>"
|
| 50 |
+
}
|