HappyBulingling commited on
Commit
72f5707
·
verified ·
1 Parent(s): 09a1a87

Upload MyModel checkpoint

Browse files
Files changed (14) hide show
  1. README.md +28 -1
  2. attention.py +124 -0
  3. block.py +65 -0
  4. chat_template.jinja +1 -0
  5. config.json +36 -0
  6. config.py +127 -0
  7. generation_config.json +7 -0
  8. model.py +287 -0
  9. model.safetensors +3 -0
  10. norm.py +19 -0
  11. rope.py +39 -0
  12. special_tokens_map.json +53 -0
  13. tokenizer.json +0 -0
  14. tokenizer_config.json +1064 -0
README.md CHANGED
@@ -1,3 +1,30 @@
1
  ---
2
- license: mit
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ library_name: transformers
3
+ pipeline_tag: text-generation
4
+ tags:
5
+ - mymodel
6
+ - causal-lm
7
  ---
8
+
9
+ # MyModel-sft
10
+
11
+ MyModel 稠密文本模型(sft-4k),参数量 259M,训练 step 4,968,tokens 0.17B。
12
+
13
+ ## 使用
14
+
15
+ ```python
16
+ from transformers import AutoModelForCausalLM, AutoTokenizer
17
+
18
+ repo_id = "HappyBulingling/MyModel-sft"
19
+ tokenizer = AutoTokenizer.from_pretrained(repo_id)
20
+ model = AutoModelForCausalLM.from_pretrained(repo_id, trust_remote_code=True)
21
+
22
+ messages = [{"role": "user", "content": "你好"}]
23
+ inputs = tokenizer.apply_chat_template(
24
+ messages, add_generation_prompt=True, return_tensors="pt", enable_thinking=True
25
+ )
26
+ outputs = model.generate(
27
+ inputs, max_new_tokens=256, do_sample=True, temperature=0.8, top_p=0.9
28
+ )
29
+ print(tokenizer.decode(outputs[0][inputs.shape[-1]:], skip_special_tokens=True))
30
+ ```
attention.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Grouped-query attention with q/k RMSNorm and SDPA."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+ import torch.nn.functional as F
7
+ from torch import nn
8
+
9
+ from .config import MyModelConfig
10
+ from .norm import RMSNorm
11
+ from .rope import apply_rotary_pos_emb
12
+
13
+
14
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
15
+ if n_rep == 1:
16
+ return hidden_states
17
+ batch, seq_len, num_kv_heads, head_dim = hidden_states.shape
18
+ hidden_states = hidden_states[:, :, :, None, :].expand(
19
+ batch, seq_len, num_kv_heads, n_rep, head_dim
20
+ )
21
+ return hidden_states.reshape(batch, seq_len, num_kv_heads * n_rep, head_dim)
22
+
23
+
24
+ def causal_mask(
25
+ q_len: int,
26
+ kv_len: int,
27
+ device: torch.device,
28
+ ) -> torch.Tensor:
29
+ """Boolean mask for a query block aligned to the end of the KV cache."""
30
+ rows = torch.arange(q_len, device=device).unsqueeze(1) + (kv_len - q_len)
31
+ cols = torch.arange(kv_len, device=device).unsqueeze(0)
32
+ return cols <= rows
33
+
34
+
35
+ class Attention(nn.Module):
36
+ def __init__(self, config: MyModelConfig):
37
+ super().__init__()
38
+ self.num_heads = config.num_attention_heads
39
+ self.num_kv_heads = config.num_key_value_heads
40
+ self.head_dim = config.head_dim
41
+ self.num_kv_groups = self.num_heads // self.num_kv_heads
42
+ self.dropout = config.dropout
43
+
44
+ self.q_proj = nn.Linear(
45
+ config.hidden_size,
46
+ self.num_heads * self.head_dim,
47
+ bias=config.attention_bias,
48
+ )
49
+ self.k_proj = nn.Linear(
50
+ config.hidden_size,
51
+ self.num_kv_heads * self.head_dim,
52
+ bias=config.attention_bias,
53
+ )
54
+ self.v_proj = nn.Linear(
55
+ config.hidden_size,
56
+ self.num_kv_heads * self.head_dim,
57
+ bias=config.attention_bias,
58
+ )
59
+ self.o_proj = nn.Linear(
60
+ self.num_heads * self.head_dim,
61
+ config.hidden_size,
62
+ bias=config.attention_bias,
63
+ )
64
+ self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
65
+ self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
66
+
67
+ def forward(
68
+ self,
69
+ hidden_states: torch.Tensor,
70
+ cos: torch.Tensor,
71
+ sin: torch.Tensor,
72
+ past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
73
+ use_cache: bool = False,
74
+ attention_mask: torch.Tensor | None = None,
75
+ ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
76
+ batch, seq_len, _ = hidden_states.shape
77
+
78
+ query = self.q_proj(hidden_states).view(batch, seq_len, self.num_heads, self.head_dim)
79
+ key = self.k_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
80
+ value = self.v_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
81
+
82
+ query = self.q_norm(query)
83
+ key = self.k_norm(key)
84
+ query, key = apply_rotary_pos_emb(query, key, cos, sin)
85
+
86
+ if past_key_value is not None:
87
+ key = torch.cat((past_key_value[0], key), dim=1)
88
+ value = torch.cat((past_key_value[1], value), dim=1)
89
+
90
+ present = (key, value) if use_cache else None
91
+ kv_len = key.shape[1]
92
+
93
+ key = repeat_kv(key, self.num_kv_groups).transpose(1, 2)
94
+ value = repeat_kv(value, self.num_kv_groups).transpose(1, 2)
95
+ query = query.transpose(1, 2)
96
+
97
+ dropout_p = self.dropout if self.training else 0.0
98
+ if attention_mask is None and past_key_value is None:
99
+ output = F.scaled_dot_product_attention(
100
+ query,
101
+ key,
102
+ value,
103
+ dropout_p=dropout_p,
104
+ is_causal=True,
105
+ )
106
+ else:
107
+ mask = causal_mask(seq_len, kv_len, hidden_states.device)
108
+ if attention_mask is not None:
109
+ if attention_mask.shape != (batch, kv_len):
110
+ raise ValueError(
111
+ f"attention_mask must have shape {(batch, kv_len)}, "
112
+ f"got {tuple(attention_mask.shape)}"
113
+ )
114
+ mask = mask.unsqueeze(0).unsqueeze(0) & attention_mask[:, None, None, :].bool()
115
+ output = F.scaled_dot_product_attention(
116
+ query,
117
+ key,
118
+ value,
119
+ attn_mask=mask,
120
+ dropout_p=dropout_p,
121
+ )
122
+
123
+ output = output.transpose(1, 2).reshape(batch, seq_len, -1)
124
+ return self.o_proj(output), present
block.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Transformer block: pre-norm GQA + SwiGLU with two residual paths."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+ from torch import nn
7
+ from transformers.activations import ACT2FN
8
+
9
+ from .attention import Attention
10
+ from .config import MyModelConfig
11
+ from .norm import RMSNorm
12
+
13
+
14
+ class FeedForward(nn.Module):
15
+ def __init__(self, config: MyModelConfig):
16
+ super().__init__()
17
+ self.gate_proj = nn.Linear(
18
+ config.hidden_size,
19
+ config.intermediate_size,
20
+ bias=config.mlp_bias,
21
+ )
22
+ self.up_proj = nn.Linear(
23
+ config.hidden_size,
24
+ config.intermediate_size,
25
+ bias=config.mlp_bias,
26
+ )
27
+ self.down_proj = nn.Linear(
28
+ config.intermediate_size,
29
+ config.hidden_size,
30
+ bias=config.mlp_bias,
31
+ )
32
+ self.act_fn = ACT2FN[config.hidden_act]
33
+
34
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
35
+ return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
36
+
37
+
38
+ class Block(nn.Module):
39
+ def __init__(self, config: MyModelConfig):
40
+ super().__init__()
41
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
42
+ self.self_attn = Attention(config)
43
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
44
+ self.mlp = FeedForward(config)
45
+
46
+ def forward(
47
+ self,
48
+ hidden_states: torch.Tensor,
49
+ cos: torch.Tensor,
50
+ sin: torch.Tensor,
51
+ past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
52
+ use_cache: bool = False,
53
+ attention_mask: torch.Tensor | None = None,
54
+ ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
55
+ attention_output, present = self.self_attn(
56
+ self.input_layernorm(hidden_states),
57
+ cos,
58
+ sin,
59
+ past_key_value,
60
+ use_cache,
61
+ attention_mask,
62
+ )
63
+ hidden_states = hidden_states + attention_output
64
+ hidden_states = hidden_states + self.mlp(self.post_attention_layernorm(hidden_states))
65
+ return hidden_states, present
chat_template.jinja ADDED
@@ -0,0 +1 @@
 
 
1
+ {% for message in messages %}{{'<|im_start|>' + message['role'] + '\n'}}{% if message['role'] == 'assistant' %}{% if message.get('reasoning_content') %}{{'<think>\n' + message['reasoning_content'] + '\n</think>\n\n' + message['content']}}{% else %}{{'<think>\n\n</think>\n\n' + message['content']}}{% endif %}{% else %}{{message['content']}}{% endif %}{{'<|im_end|>\n'}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant\n'}}{% if enable_thinking is defined and not enable_thinking %}{{'<think>\n\n</think>\n\n'}}{% else %}{{'<think>'}}{% endif %}{% endif %}
config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MyModelForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "auto_map": {
7
+ "AutoConfig": "config.MyModelConfig",
8
+ "AutoModelForCausalLM": "model.MyModelForCausalLM"
9
+ },
10
+ "bos_token_id": 2,
11
+ "dropout": 0.0,
12
+ "dtype": "float32",
13
+ "eos_token_id": 0,
14
+ "ffn_type": "dense",
15
+ "head_dim": 64,
16
+ "hidden_act": "silu",
17
+ "hidden_size": 1024,
18
+ "im_end_token_id": 5,
19
+ "initializer_range": 0.02,
20
+ "intermediate_size": 2816,
21
+ "max_position_embeddings": 4096,
22
+ "mlp_bias": false,
23
+ "model_type": "mymodel",
24
+ "num_attention_heads": 16,
25
+ "num_hidden_layers": 20,
26
+ "num_key_value_heads": 4,
27
+ "pad_token_id": 1,
28
+ "parametrization": "standard",
29
+ "qk_norm": true,
30
+ "rms_norm_eps": 1e-06,
31
+ "rope_scaling": null,
32
+ "rope_theta": 1000000.0,
33
+ "transformers_version": "4.57.6",
34
+ "unk_token_id": 3,
35
+ "vocab_size": 32768
36
+ }
config.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Model configuration and the two presets used by this project."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from transformers import PretrainedConfig
6
+
7
+
8
+ PRESETS = {
9
+ "tiny": {
10
+ "hidden_size": 256,
11
+ "num_hidden_layers": 4,
12
+ "num_attention_heads": 8,
13
+ "num_key_value_heads": 1,
14
+ "head_dim": 32,
15
+ "intermediate_size": 896,
16
+ "vocab_size": 32768,
17
+ },
18
+ "base": {
19
+ "hidden_size": 1024,
20
+ "num_hidden_layers": 20,
21
+ "num_attention_heads": 16,
22
+ "num_key_value_heads": 4,
23
+ "head_dim": 64,
24
+ "intermediate_size": 2816,
25
+ "vocab_size": 32768,
26
+ },
27
+ }
28
+
29
+
30
+ class MyModelConfig(PretrainedConfig):
31
+ """Configuration for the 259M dense text backbone.
32
+
33
+ Defaults: 20 layers, d=1024, GQA 16:4, SwiGLU 2816, 32K vocabulary,
34
+ tied embeddings.
35
+ """
36
+
37
+ model_type = "mymodel"
38
+
39
+ def __init__(
40
+ self,
41
+ hidden_size: int = 1024,
42
+ num_hidden_layers: int = 20,
43
+ num_attention_heads: int = 16,
44
+ num_key_value_heads: int = 4,
45
+ head_dim: int = 64,
46
+ intermediate_size: int = 2816,
47
+ vocab_size: int = 32768,
48
+ hidden_act: str = "silu",
49
+ rms_norm_eps: float = 1e-6,
50
+ qk_norm: bool = True,
51
+ rope_theta: float = 1_000_000.0,
52
+ max_position_embeddings: int = 4096,
53
+ rope_scaling: dict | None = None,
54
+ tie_word_embeddings: bool = True,
55
+ attention_bias: bool = False,
56
+ mlp_bias: bool = False,
57
+ dropout: float = 0.0,
58
+ ffn_type: str = "dense",
59
+ parametrization: str = "standard",
60
+ initializer_range: float = 0.02,
61
+ bos_token_id: int = 2,
62
+ eos_token_id: int = 0,
63
+ im_end_token_id: int = 5,
64
+ pad_token_id: int = 1,
65
+ unk_token_id: int = 3,
66
+ **kwargs,
67
+ ):
68
+ super().__init__(
69
+ bos_token_id=bos_token_id,
70
+ eos_token_id=eos_token_id,
71
+ pad_token_id=pad_token_id,
72
+ unk_token_id=unk_token_id,
73
+ tie_word_embeddings=tie_word_embeddings,
74
+ **kwargs,
75
+ )
76
+ self.im_end_token_id = im_end_token_id
77
+ if hidden_size % num_attention_heads != 0:
78
+ raise ValueError("hidden_size must be divisible by num_attention_heads")
79
+ if num_attention_heads % num_key_value_heads != 0:
80
+ raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
81
+ if head_dim * num_attention_heads != hidden_size:
82
+ raise ValueError("head_dim * num_attention_heads must equal hidden_size")
83
+
84
+ self.hidden_size = hidden_size
85
+ self.num_hidden_layers = num_hidden_layers
86
+ self.num_attention_heads = num_attention_heads
87
+ self.num_key_value_heads = num_key_value_heads
88
+ self.head_dim = head_dim
89
+ self.intermediate_size = intermediate_size
90
+ self.vocab_size = vocab_size
91
+ self.hidden_act = hidden_act
92
+ self.rms_norm_eps = rms_norm_eps
93
+ self.qk_norm = qk_norm
94
+ self.rope_theta = rope_theta
95
+ self.max_position_embeddings = max_position_embeddings
96
+ self.rope_scaling = rope_scaling
97
+ self.attention_bias = attention_bias
98
+ self.mlp_bias = mlp_bias
99
+ self.dropout = dropout
100
+ self.ffn_type = ffn_type
101
+ self.parametrization = parametrization
102
+ self.initializer_range = initializer_range
103
+
104
+ if self.ffn_type != "dense":
105
+ raise ValueError(f"unsupported ffn_type: {self.ffn_type}")
106
+ if self.parametrization != "standard":
107
+ raise ValueError(f"unsupported parametrization: {self.parametrization}")
108
+
109
+ @classmethod
110
+ def from_preset(cls, name: str, **overrides) -> "MyModelConfig":
111
+ if name not in PRESETS:
112
+ raise ValueError(f"unknown preset {name!r}; choose from {sorted(PRESETS)}")
113
+ values = dict(PRESETS[name])
114
+ values.update(overrides)
115
+ return cls(**values)
116
+
117
+ def estimate_parameters(self) -> int:
118
+ """Formula-based parameter count, excluding q/k norm parameters."""
119
+ d = self.hidden_size
120
+ attn = 2 * d * (self.num_attention_heads * self.head_dim)
121
+ attn += 2 * d * (self.num_key_value_heads * self.head_dim)
122
+ ffn = 3 * d * self.intermediate_size
123
+ norms = self.num_hidden_layers * 2 * d + d
124
+ if self.qk_norm:
125
+ norms += self.num_hidden_layers * 2 * self.head_dim
126
+ embedding = self.vocab_size * d
127
+ return self.num_hidden_layers * (attn + ffn) + norms + embedding
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 2,
4
+ "eos_token_id": 0,
5
+ "pad_token_id": 1,
6
+ "transformers_version": "4.57.6"
7
+ }
model.py ADDED
@@ -0,0 +1,287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MyModel backbone and causal language-model head."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Sequence
7
+ from typing import Iterable
8
+
9
+ import torch
10
+ import torch.nn.functional as F
11
+ from torch import nn
12
+ from torch.utils.checkpoint import checkpoint
13
+ from transformers import GenerationMixin, PreTrainedModel
14
+ from transformers.modeling_outputs import CausalLMOutputWithPast
15
+
16
+ from .block import Block
17
+ from .config import MyModelConfig
18
+ from .norm import RMSNorm
19
+ from .rope import precompute_rope
20
+
21
+
22
+ def chunked_cross_entropy(
23
+ hidden_states: torch.Tensor,
24
+ labels: torch.Tensor,
25
+ lm_head: nn.Linear,
26
+ chunk_size: int,
27
+ ignore_index: int = -100,
28
+ ) -> torch.Tensor:
29
+ """Next-token cross entropy without materializing all logits at once."""
30
+ if hidden_states.shape[:2] != labels.shape:
31
+ raise ValueError(
32
+ f"hidden_states {tuple(hidden_states.shape[:2])} and labels "
33
+ f"{tuple(labels.shape)} must agree on batch and sequence length"
34
+ )
35
+ if chunk_size <= 0:
36
+ raise ValueError("chunk_size must be positive")
37
+
38
+ hidden = hidden_states[:, :-1, :]
39
+ target = labels[:, 1:]
40
+ total_loss = hidden_states.new_zeros(())
41
+ total_tokens = 0
42
+ for start in range(0, hidden.shape[1], chunk_size):
43
+ chunk_hidden = hidden[:, start : start + chunk_size, :]
44
+ chunk_target = target[:, start : start + chunk_size]
45
+ logits = lm_head(chunk_hidden)
46
+ total_loss = total_loss + F.cross_entropy(
47
+ logits.reshape(-1, logits.shape[-1]),
48
+ chunk_target.reshape(-1),
49
+ ignore_index=ignore_index,
50
+ reduction="sum",
51
+ )
52
+ total_tokens += int((chunk_target != ignore_index).sum())
53
+ return total_loss / max(total_tokens, 1)
54
+
55
+
56
+ class MyModelModel(nn.Module):
57
+ def __init__(self, config: MyModelConfig):
58
+ super().__init__()
59
+ self.config = config
60
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
61
+ self.dropout = nn.Dropout(config.dropout)
62
+ self.layers = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)])
63
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
64
+
65
+ cos, sin = precompute_rope(
66
+ config.head_dim,
67
+ config.max_position_embeddings,
68
+ config.rope_theta,
69
+ )
70
+ self.register_buffer("rope_cos", cos, persistent=False)
71
+ self.register_buffer("rope_sin", sin, persistent=False)
72
+ self.gradient_checkpointing = False
73
+
74
+ def gradient_checkpointing_enable(self) -> None:
75
+ self.gradient_checkpointing = True
76
+
77
+ def gradient_checkpointing_disable(self) -> None:
78
+ self.gradient_checkpointing = False
79
+
80
+ def forward(
81
+ self,
82
+ input_ids: torch.Tensor,
83
+ attention_mask: torch.Tensor | None = None,
84
+ past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None,
85
+ use_cache: bool = False,
86
+ ) -> tuple[torch.Tensor, tuple[tuple[torch.Tensor, torch.Tensor], ...] | None]:
87
+ batch, seq_len = input_ids.shape
88
+ if past_key_values is None:
89
+ past_key_values = [None] * len(self.layers)
90
+ else:
91
+ past_key_values = list(past_key_values)
92
+ if len(past_key_values) != len(self.layers):
93
+ raise ValueError(
94
+ f"expected {len(self.layers)} past key/value entries, "
95
+ f"got {len(past_key_values)}"
96
+ )
97
+
98
+ start_pos = 0 if past_key_values[0] is None else past_key_values[0][0].shape[1]
99
+ if start_pos + seq_len > self.config.max_position_embeddings:
100
+ raise ValueError(
101
+ f"sequence length {start_pos + seq_len} exceeds "
102
+ f"max_position_embeddings {self.config.max_position_embeddings}"
103
+ )
104
+
105
+ hidden_states = self.dropout(self.embed_tokens(input_ids))
106
+ cos = self.rope_cos[start_pos : start_pos + seq_len]
107
+ sin = self.rope_sin[start_pos : start_pos + seq_len]
108
+
109
+ presents: list[tuple[torch.Tensor, torch.Tensor] | None] = []
110
+ for layer, past_key_value in zip(self.layers, past_key_values):
111
+ if self.gradient_checkpointing and self.training:
112
+ hidden_states, _ = checkpoint(
113
+ layer,
114
+ hidden_states,
115
+ cos,
116
+ sin,
117
+ past_key_value,
118
+ False,
119
+ attention_mask,
120
+ use_reentrant=False,
121
+ )
122
+ presents.append(None)
123
+ else:
124
+ hidden_states, present = layer(
125
+ hidden_states,
126
+ cos,
127
+ sin,
128
+ past_key_value,
129
+ use_cache,
130
+ attention_mask,
131
+ )
132
+ presents.append(present)
133
+
134
+ hidden_states = self.norm(hidden_states)
135
+ if not use_cache:
136
+ return hidden_states, None
137
+ return hidden_states, tuple(presents)
138
+
139
+
140
+ class MyModelForCausalLM(PreTrainedModel, GenerationMixin):
141
+ config_class = MyModelConfig
142
+ _tied_weights_keys = ["lm_head.weight"]
143
+
144
+ def __init__(self, config: MyModelConfig):
145
+ super().__init__(config)
146
+ self.model = MyModelModel(config)
147
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
148
+ if config.tie_word_embeddings:
149
+ self.lm_head.weight = self.model.embed_tokens.weight
150
+ self.post_init()
151
+ self._scale_residual_projections()
152
+
153
+ def _init_weights(self, module: nn.Module) -> None:
154
+ if isinstance(module, nn.Linear):
155
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
156
+ if module.bias is not None:
157
+ module.bias.data.zero_()
158
+ elif isinstance(module, nn.Embedding):
159
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
160
+
161
+ def _scale_residual_projections(self) -> None:
162
+ scale = 1.0 / math.sqrt(2 * self.config.num_hidden_layers)
163
+ with torch.no_grad():
164
+ for layer in self.model.layers:
165
+ layer.self_attn.o_proj.weight.mul_(scale)
166
+ layer.mlp.down_proj.weight.mul_(scale)
167
+
168
+ def gradient_checkpointing_enable(self) -> None:
169
+ self.model.gradient_checkpointing_enable()
170
+
171
+ def gradient_checkpointing_disable(self) -> None:
172
+ self.model.gradient_checkpointing_disable()
173
+
174
+ def forward(
175
+ self,
176
+ input_ids: torch.Tensor,
177
+ attention_mask: torch.Tensor | None = None,
178
+ past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None,
179
+ use_cache: bool = False,
180
+ labels: torch.Tensor | None = None,
181
+ chunk_size: int = 0,
182
+ logits_to_keep: int = 0,
183
+ **kwargs,
184
+ ) -> CausalLMOutputWithPast:
185
+ hidden_states, presents = self.model(
186
+ input_ids,
187
+ attention_mask=attention_mask,
188
+ past_key_values=past_key_values,
189
+ use_cache=use_cache,
190
+ )
191
+
192
+ loss = None
193
+ logits = None
194
+ if labels is not None and chunk_size and chunk_size > 0:
195
+ loss = chunked_cross_entropy(hidden_states, labels, self.lm_head, chunk_size)
196
+ else:
197
+ logits = self.lm_head(hidden_states)
198
+ if labels is not None:
199
+ shift_logits = logits[:, :-1, :]
200
+ shift_labels = labels[:, 1:]
201
+ loss = F.cross_entropy(
202
+ shift_logits.reshape(-1, shift_logits.shape[-1]),
203
+ shift_labels.reshape(-1),
204
+ ignore_index=-100,
205
+ )
206
+ if logits_to_keep > 0:
207
+ logits = logits[:, -logits_to_keep:, :]
208
+
209
+ return CausalLMOutputWithPast(
210
+ loss=loss,
211
+ logits=logits,
212
+ past_key_values=presents,
213
+ hidden_states=hidden_states,
214
+ attentions=None,
215
+ )
216
+
217
+ @torch.inference_mode()
218
+ def generate(
219
+ self,
220
+ input_ids: torch.Tensor,
221
+ max_new_tokens: int = 64,
222
+ temperature: float = 1.0,
223
+ top_k: int = 0,
224
+ top_p: float = 1.0,
225
+ eos_token_id: int | Sequence[int] | None = None,
226
+ do_sample: bool = True,
227
+ use_cache: bool = True,
228
+ ) -> torch.Tensor:
229
+ if temperature <= 0:
230
+ raise ValueError("temperature must be positive")
231
+ if eos_token_id is None:
232
+ stop_ids = [int(self.config.eos_token_id)]
233
+ im_end = getattr(self.config, "im_end_token_id", None)
234
+ if im_end is not None:
235
+ stop_ids.append(int(im_end))
236
+ elif isinstance(eos_token_id, int):
237
+ stop_ids = [eos_token_id]
238
+ else:
239
+ stop_ids = [int(token_id) for token_id in eos_token_id]
240
+ stop_ids = list(dict.fromkeys(stop_ids))
241
+ if not stop_ids:
242
+ raise ValueError("eos_token_id must contain at least one token id")
243
+ stop_tensor = torch.tensor(
244
+ stop_ids, dtype=torch.long, device=input_ids.device
245
+ )
246
+ fill_id = stop_ids[0]
247
+
248
+ past_key_values = None
249
+ finished = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device)
250
+ for _ in range(max_new_tokens):
251
+ past_len = 0 if past_key_values is None else past_key_values[0][0].shape[1]
252
+ outputs = self(
253
+ input_ids[:, past_len:],
254
+ past_key_values=past_key_values,
255
+ use_cache=use_cache,
256
+ )
257
+ logits = outputs.logits[:, -1, :] / temperature
258
+
259
+ if top_k > 0:
260
+ threshold = torch.topk(logits, min(top_k, logits.shape[-1]), dim=-1).values[..., -1:]
261
+ logits = logits.masked_fill(logits < threshold, float("-inf"))
262
+ if top_p < 1.0:
263
+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
264
+ cumulative = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
265
+ remove = cumulative > top_p
266
+ remove[..., 1:] = remove[..., :-1].clone()
267
+ remove[..., 0] = False
268
+ logits = logits.masked_fill(remove.scatter(1, sorted_indices, remove), float("-inf"))
269
+
270
+ if do_sample:
271
+ probs = torch.softmax(logits, dim=-1)
272
+ next_token = torch.multinomial(probs, num_samples=1)
273
+ else:
274
+ next_token = torch.argmax(logits, dim=-1, keepdim=True)
275
+
276
+ next_token = torch.where(
277
+ finished.unsqueeze(-1),
278
+ next_token.new_full((next_token.shape[0], 1), fill_id),
279
+ next_token,
280
+ )
281
+ input_ids = torch.cat((input_ids, next_token), dim=-1)
282
+ past_key_values = outputs.past_key_values if use_cache else None
283
+
284
+ finished |= (next_token == stop_tensor).any(dim=-1).reshape(-1)
285
+ if finished.all():
286
+ break
287
+ return input_ids
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7e91b5e476ad9d95071cad2a271ad79694e6e0e5fdaff1846f2575ce601589b
3
+ size 1036196104
norm.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """RMSNorm with fp32 statistics and parameter-free operations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+ from torch import nn
7
+
8
+
9
+ class RMSNorm(nn.Module):
10
+ def __init__(self, hidden_size: int, eps: float = 1e-6):
11
+ super().__init__()
12
+ self.weight = nn.Parameter(torch.ones(hidden_size))
13
+ self.eps = eps
14
+
15
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
16
+ dtype = hidden_states.dtype
17
+ x = hidden_states.float()
18
+ x = x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
19
+ return (self.weight.float() * x).to(dtype)
rope.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Rotary position embeddings."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+
8
+ def precompute_rope(
9
+ head_dim: int,
10
+ max_position_embeddings: int,
11
+ rope_theta: float,
12
+ ) -> tuple[torch.Tensor, torch.Tensor]:
13
+ if head_dim % 2 != 0:
14
+ raise ValueError("head_dim must be even for RoPE")
15
+ inv_freq = 1.0 / (
16
+ rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)
17
+ )
18
+ positions = torch.arange(max_position_embeddings, dtype=torch.float32)
19
+ freqs = torch.outer(positions, inv_freq)
20
+ emb = torch.cat((freqs, freqs), dim=-1)
21
+ return emb.cos(), emb.sin()
22
+
23
+
24
+ def rotate_half(x: torch.Tensor) -> torch.Tensor:
25
+ half = x.shape[-1] // 2
26
+ return torch.cat((-x[..., half:], x[..., :half]), dim=-1)
27
+
28
+
29
+ def apply_rotary_pos_emb(
30
+ q: torch.Tensor,
31
+ k: torch.Tensor,
32
+ cos: torch.Tensor,
33
+ sin: torch.Tensor,
34
+ ) -> tuple[torch.Tensor, torch.Tensor]:
35
+ cos = cos.to(q.device).unsqueeze(0).unsqueeze(2)
36
+ sin = sin.to(q.device).unsqueeze(0).unsqueeze(2)
37
+ q_embed = (q * cos) + (rotate_half(q) * sin)
38
+ k_embed = (k * cos) + (rotate_half(k) * sin)
39
+ return q_embed, k_embed
special_tokens_map.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|system|>",
6
+ "<|user|>",
7
+ "<|assistant|>",
8
+ "<|tool|>",
9
+ "<think>",
10
+ "</think>",
11
+ "<tool_call>",
12
+ "</tool_call>",
13
+ "<tool_response>",
14
+ "</tool_response>",
15
+ "<image_start>",
16
+ "<image_pad>",
17
+ "<image_end>",
18
+ "<audio_start>",
19
+ "<audio_pad>",
20
+ "<audio_end>",
21
+ "<speech_bos>",
22
+ "<speech_eos>",
23
+ "<speech_pad>"
24
+ ],
25
+ "bos_token": {
26
+ "content": "<|bos|>",
27
+ "lstrip": false,
28
+ "normalized": false,
29
+ "rstrip": false,
30
+ "single_word": false
31
+ },
32
+ "eos_token": {
33
+ "content": "<|eos|>",
34
+ "lstrip": false,
35
+ "normalized": false,
36
+ "rstrip": false,
37
+ "single_word": false
38
+ },
39
+ "pad_token": {
40
+ "content": "<|pad|>",
41
+ "lstrip": false,
42
+ "normalized": false,
43
+ "rstrip": false,
44
+ "single_word": false
45
+ },
46
+ "unk_token": {
47
+ "content": "<|unk|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": false,
51
+ "single_word": false
52
+ }
53
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,1064 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": false,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<|eos|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<|pad|>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "<|bos|>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "3": {
31
+ "content": "<|unk|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "4": {
39
+ "content": "<|im_start|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "5": {
47
+ "content": "<|im_end|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": false,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "6": {
55
+ "content": "<|system|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": false,
59
+ "single_word": false,
60
+ "special": true
61
+ },
62
+ "7": {
63
+ "content": "<|user|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": false,
67
+ "single_word": false,
68
+ "special": true
69
+ },
70
+ "8": {
71
+ "content": "<|assistant|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": false,
75
+ "single_word": false,
76
+ "special": true
77
+ },
78
+ "9": {
79
+ "content": "<|tool|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": false,
83
+ "single_word": false,
84
+ "special": true
85
+ },
86
+ "10": {
87
+ "content": "<think>",
88
+ "lstrip": false,
89
+ "normalized": false,
90
+ "rstrip": false,
91
+ "single_word": false,
92
+ "special": true
93
+ },
94
+ "11": {
95
+ "content": "</think>",
96
+ "lstrip": false,
97
+ "normalized": false,
98
+ "rstrip": false,
99
+ "single_word": false,
100
+ "special": true
101
+ },
102
+ "12": {
103
+ "content": "<tool_call>",
104
+ "lstrip": false,
105
+ "normalized": false,
106
+ "rstrip": false,
107
+ "single_word": false,
108
+ "special": true
109
+ },
110
+ "13": {
111
+ "content": "</tool_call>",
112
+ "lstrip": false,
113
+ "normalized": false,
114
+ "rstrip": false,
115
+ "single_word": false,
116
+ "special": true
117
+ },
118
+ "14": {
119
+ "content": "<tool_response>",
120
+ "lstrip": false,
121
+ "normalized": false,
122
+ "rstrip": false,
123
+ "single_word": false,
124
+ "special": true
125
+ },
126
+ "15": {
127
+ "content": "</tool_response>",
128
+ "lstrip": false,
129
+ "normalized": false,
130
+ "rstrip": false,
131
+ "single_word": false,
132
+ "special": true
133
+ },
134
+ "16": {
135
+ "content": "<image_start>",
136
+ "lstrip": false,
137
+ "normalized": false,
138
+ "rstrip": false,
139
+ "single_word": false,
140
+ "special": true
141
+ },
142
+ "17": {
143
+ "content": "<image_pad>",
144
+ "lstrip": false,
145
+ "normalized": false,
146
+ "rstrip": false,
147
+ "single_word": false,
148
+ "special": true
149
+ },
150
+ "18": {
151
+ "content": "<image_end>",
152
+ "lstrip": false,
153
+ "normalized": false,
154
+ "rstrip": false,
155
+ "single_word": false,
156
+ "special": true
157
+ },
158
+ "19": {
159
+ "content": "<audio_start>",
160
+ "lstrip": false,
161
+ "normalized": false,
162
+ "rstrip": false,
163
+ "single_word": false,
164
+ "special": true
165
+ },
166
+ "20": {
167
+ "content": "<audio_pad>",
168
+ "lstrip": false,
169
+ "normalized": false,
170
+ "rstrip": false,
171
+ "single_word": false,
172
+ "special": true
173
+ },
174
+ "21": {
175
+ "content": "<audio_end>",
176
+ "lstrip": false,
177
+ "normalized": false,
178
+ "rstrip": false,
179
+ "single_word": false,
180
+ "special": true
181
+ },
182
+ "22": {
183
+ "content": "<speech_bos>",
184
+ "lstrip": false,
185
+ "normalized": false,
186
+ "rstrip": false,
187
+ "single_word": false,
188
+ "special": true
189
+ },
190
+ "23": {
191
+ "content": "<speech_eos>",
192
+ "lstrip": false,
193
+ "normalized": false,
194
+ "rstrip": false,
195
+ "single_word": false,
196
+ "special": true
197
+ },
198
+ "24": {
199
+ "content": "<speech_pad>",
200
+ "lstrip": false,
201
+ "normalized": false,
202
+ "rstrip": false,
203
+ "single_word": false,
204
+ "special": true
205
+ },
206
+ "25": {
207
+ "content": "<|reserved_1|>",
208
+ "lstrip": false,
209
+ "normalized": false,
210
+ "rstrip": false,
211
+ "single_word": false,
212
+ "special": true
213
+ },
214
+ "26": {
215
+ "content": "<|reserved_2|>",
216
+ "lstrip": false,
217
+ "normalized": false,
218
+ "rstrip": false,
219
+ "single_word": false,
220
+ "special": true
221
+ },
222
+ "27": {
223
+ "content": "<|reserved_3|>",
224
+ "lstrip": false,
225
+ "normalized": false,
226
+ "rstrip": false,
227
+ "single_word": false,
228
+ "special": true
229
+ },
230
+ "28": {
231
+ "content": "<|reserved_4|>",
232
+ "lstrip": false,
233
+ "normalized": false,
234
+ "rstrip": false,
235
+ "single_word": false,
236
+ "special": true
237
+ },
238
+ "29": {
239
+ "content": "<|reserved_5|>",
240
+ "lstrip": false,
241
+ "normalized": false,
242
+ "rstrip": false,
243
+ "single_word": false,
244
+ "special": true
245
+ },
246
+ "30": {
247
+ "content": "<|reserved_6|>",
248
+ "lstrip": false,
249
+ "normalized": false,
250
+ "rstrip": false,
251
+ "single_word": false,
252
+ "special": true
253
+ },
254
+ "31": {
255
+ "content": "<|reserved_7|>",
256
+ "lstrip": false,
257
+ "normalized": false,
258
+ "rstrip": false,
259
+ "single_word": false,
260
+ "special": true
261
+ },
262
+ "32": {
263
+ "content": "<|reserved_8|>",
264
+ "lstrip": false,
265
+ "normalized": false,
266
+ "rstrip": false,
267
+ "single_word": false,
268
+ "special": true
269
+ },
270
+ "33": {
271
+ "content": "<|reserved_9|>",
272
+ "lstrip": false,
273
+ "normalized": false,
274
+ "rstrip": false,
275
+ "single_word": false,
276
+ "special": true
277
+ },
278
+ "34": {
279
+ "content": "<|reserved_10|>",
280
+ "lstrip": false,
281
+ "normalized": false,
282
+ "rstrip": false,
283
+ "single_word": false,
284
+ "special": true
285
+ },
286
+ "35": {
287
+ "content": "<|reserved_11|>",
288
+ "lstrip": false,
289
+ "normalized": false,
290
+ "rstrip": false,
291
+ "single_word": false,
292
+ "special": true
293
+ },
294
+ "36": {
295
+ "content": "<|reserved_12|>",
296
+ "lstrip": false,
297
+ "normalized": false,
298
+ "rstrip": false,
299
+ "single_word": false,
300
+ "special": true
301
+ },
302
+ "37": {
303
+ "content": "<|reserved_13|>",
304
+ "lstrip": false,
305
+ "normalized": false,
306
+ "rstrip": false,
307
+ "single_word": false,
308
+ "special": true
309
+ },
310
+ "38": {
311
+ "content": "<|reserved_14|>",
312
+ "lstrip": false,
313
+ "normalized": false,
314
+ "rstrip": false,
315
+ "single_word": false,
316
+ "special": true
317
+ },
318
+ "39": {
319
+ "content": "<|reserved_15|>",
320
+ "lstrip": false,
321
+ "normalized": false,
322
+ "rstrip": false,
323
+ "single_word": false,
324
+ "special": true
325
+ },
326
+ "40": {
327
+ "content": "<|reserved_16|>",
328
+ "lstrip": false,
329
+ "normalized": false,
330
+ "rstrip": false,
331
+ "single_word": false,
332
+ "special": true
333
+ },
334
+ "41": {
335
+ "content": "<|reserved_17|>",
336
+ "lstrip": false,
337
+ "normalized": false,
338
+ "rstrip": false,
339
+ "single_word": false,
340
+ "special": true
341
+ },
342
+ "42": {
343
+ "content": "<|reserved_18|>",
344
+ "lstrip": false,
345
+ "normalized": false,
346
+ "rstrip": false,
347
+ "single_word": false,
348
+ "special": true
349
+ },
350
+ "43": {
351
+ "content": "<|reserved_19|>",
352
+ "lstrip": false,
353
+ "normalized": false,
354
+ "rstrip": false,
355
+ "single_word": false,
356
+ "special": true
357
+ },
358
+ "44": {
359
+ "content": "<|reserved_20|>",
360
+ "lstrip": false,
361
+ "normalized": false,
362
+ "rstrip": false,
363
+ "single_word": false,
364
+ "special": true
365
+ },
366
+ "45": {
367
+ "content": "<|reserved_21|>",
368
+ "lstrip": false,
369
+ "normalized": false,
370
+ "rstrip": false,
371
+ "single_word": false,
372
+ "special": true
373
+ },
374
+ "46": {
375
+ "content": "<|reserved_22|>",
376
+ "lstrip": false,
377
+ "normalized": false,
378
+ "rstrip": false,
379
+ "single_word": false,
380
+ "special": true
381
+ },
382
+ "47": {
383
+ "content": "<|reserved_23|>",
384
+ "lstrip": false,
385
+ "normalized": false,
386
+ "rstrip": false,
387
+ "single_word": false,
388
+ "special": true
389
+ },
390
+ "48": {
391
+ "content": "<|reserved_24|>",
392
+ "lstrip": false,
393
+ "normalized": false,
394
+ "rstrip": false,
395
+ "single_word": false,
396
+ "special": true
397
+ },
398
+ "49": {
399
+ "content": "<|reserved_25|>",
400
+ "lstrip": false,
401
+ "normalized": false,
402
+ "rstrip": false,
403
+ "single_word": false,
404
+ "special": true
405
+ },
406
+ "50": {
407
+ "content": "<|reserved_26|>",
408
+ "lstrip": false,
409
+ "normalized": false,
410
+ "rstrip": false,
411
+ "single_word": false,
412
+ "special": true
413
+ },
414
+ "51": {
415
+ "content": "<|reserved_27|>",
416
+ "lstrip": false,
417
+ "normalized": false,
418
+ "rstrip": false,
419
+ "single_word": false,
420
+ "special": true
421
+ },
422
+ "52": {
423
+ "content": "<|reserved_28|>",
424
+ "lstrip": false,
425
+ "normalized": false,
426
+ "rstrip": false,
427
+ "single_word": false,
428
+ "special": true
429
+ },
430
+ "53": {
431
+ "content": "<|reserved_29|>",
432
+ "lstrip": false,
433
+ "normalized": false,
434
+ "rstrip": false,
435
+ "single_word": false,
436
+ "special": true
437
+ },
438
+ "54": {
439
+ "content": "<|reserved_30|>",
440
+ "lstrip": false,
441
+ "normalized": false,
442
+ "rstrip": false,
443
+ "single_word": false,
444
+ "special": true
445
+ },
446
+ "55": {
447
+ "content": "<|reserved_31|>",
448
+ "lstrip": false,
449
+ "normalized": false,
450
+ "rstrip": false,
451
+ "single_word": false,
452
+ "special": true
453
+ },
454
+ "56": {
455
+ "content": "<|reserved_32|>",
456
+ "lstrip": false,
457
+ "normalized": false,
458
+ "rstrip": false,
459
+ "single_word": false,
460
+ "special": true
461
+ },
462
+ "57": {
463
+ "content": "<|reserved_33|>",
464
+ "lstrip": false,
465
+ "normalized": false,
466
+ "rstrip": false,
467
+ "single_word": false,
468
+ "special": true
469
+ },
470
+ "58": {
471
+ "content": "<|reserved_34|>",
472
+ "lstrip": false,
473
+ "normalized": false,
474
+ "rstrip": false,
475
+ "single_word": false,
476
+ "special": true
477
+ },
478
+ "59": {
479
+ "content": "<|reserved_35|>",
480
+ "lstrip": false,
481
+ "normalized": false,
482
+ "rstrip": false,
483
+ "single_word": false,
484
+ "special": true
485
+ },
486
+ "60": {
487
+ "content": "<|reserved_36|>",
488
+ "lstrip": false,
489
+ "normalized": false,
490
+ "rstrip": false,
491
+ "single_word": false,
492
+ "special": true
493
+ },
494
+ "61": {
495
+ "content": "<|reserved_37|>",
496
+ "lstrip": false,
497
+ "normalized": false,
498
+ "rstrip": false,
499
+ "single_word": false,
500
+ "special": true
501
+ },
502
+ "62": {
503
+ "content": "<|reserved_38|>",
504
+ "lstrip": false,
505
+ "normalized": false,
506
+ "rstrip": false,
507
+ "single_word": false,
508
+ "special": true
509
+ },
510
+ "63": {
511
+ "content": "<|reserved_39|>",
512
+ "lstrip": false,
513
+ "normalized": false,
514
+ "rstrip": false,
515
+ "single_word": false,
516
+ "special": true
517
+ },
518
+ "64": {
519
+ "content": "<|reserved_40|>",
520
+ "lstrip": false,
521
+ "normalized": false,
522
+ "rstrip": false,
523
+ "single_word": false,
524
+ "special": true
525
+ },
526
+ "65": {
527
+ "content": "<|reserved_41|>",
528
+ "lstrip": false,
529
+ "normalized": false,
530
+ "rstrip": false,
531
+ "single_word": false,
532
+ "special": true
533
+ },
534
+ "66": {
535
+ "content": "<|reserved_42|>",
536
+ "lstrip": false,
537
+ "normalized": false,
538
+ "rstrip": false,
539
+ "single_word": false,
540
+ "special": true
541
+ },
542
+ "67": {
543
+ "content": "<|reserved_43|>",
544
+ "lstrip": false,
545
+ "normalized": false,
546
+ "rstrip": false,
547
+ "single_word": false,
548
+ "special": true
549
+ },
550
+ "68": {
551
+ "content": "<|reserved_44|>",
552
+ "lstrip": false,
553
+ "normalized": false,
554
+ "rstrip": false,
555
+ "single_word": false,
556
+ "special": true
557
+ },
558
+ "69": {
559
+ "content": "<|reserved_45|>",
560
+ "lstrip": false,
561
+ "normalized": false,
562
+ "rstrip": false,
563
+ "single_word": false,
564
+ "special": true
565
+ },
566
+ "70": {
567
+ "content": "<|reserved_46|>",
568
+ "lstrip": false,
569
+ "normalized": false,
570
+ "rstrip": false,
571
+ "single_word": false,
572
+ "special": true
573
+ },
574
+ "71": {
575
+ "content": "<|reserved_47|>",
576
+ "lstrip": false,
577
+ "normalized": false,
578
+ "rstrip": false,
579
+ "single_word": false,
580
+ "special": true
581
+ },
582
+ "72": {
583
+ "content": "<|reserved_48|>",
584
+ "lstrip": false,
585
+ "normalized": false,
586
+ "rstrip": false,
587
+ "single_word": false,
588
+ "special": true
589
+ },
590
+ "73": {
591
+ "content": "<|reserved_49|>",
592
+ "lstrip": false,
593
+ "normalized": false,
594
+ "rstrip": false,
595
+ "single_word": false,
596
+ "special": true
597
+ },
598
+ "74": {
599
+ "content": "<|reserved_50|>",
600
+ "lstrip": false,
601
+ "normalized": false,
602
+ "rstrip": false,
603
+ "single_word": false,
604
+ "special": true
605
+ },
606
+ "75": {
607
+ "content": "<|reserved_51|>",
608
+ "lstrip": false,
609
+ "normalized": false,
610
+ "rstrip": false,
611
+ "single_word": false,
612
+ "special": true
613
+ },
614
+ "76": {
615
+ "content": "<|reserved_52|>",
616
+ "lstrip": false,
617
+ "normalized": false,
618
+ "rstrip": false,
619
+ "single_word": false,
620
+ "special": true
621
+ },
622
+ "77": {
623
+ "content": "<|reserved_53|>",
624
+ "lstrip": false,
625
+ "normalized": false,
626
+ "rstrip": false,
627
+ "single_word": false,
628
+ "special": true
629
+ },
630
+ "78": {
631
+ "content": "<|reserved_54|>",
632
+ "lstrip": false,
633
+ "normalized": false,
634
+ "rstrip": false,
635
+ "single_word": false,
636
+ "special": true
637
+ },
638
+ "79": {
639
+ "content": "<|reserved_55|>",
640
+ "lstrip": false,
641
+ "normalized": false,
642
+ "rstrip": false,
643
+ "single_word": false,
644
+ "special": true
645
+ },
646
+ "80": {
647
+ "content": "<|reserved_56|>",
648
+ "lstrip": false,
649
+ "normalized": false,
650
+ "rstrip": false,
651
+ "single_word": false,
652
+ "special": true
653
+ },
654
+ "81": {
655
+ "content": "<|reserved_57|>",
656
+ "lstrip": false,
657
+ "normalized": false,
658
+ "rstrip": false,
659
+ "single_word": false,
660
+ "special": true
661
+ },
662
+ "82": {
663
+ "content": "<|reserved_58|>",
664
+ "lstrip": false,
665
+ "normalized": false,
666
+ "rstrip": false,
667
+ "single_word": false,
668
+ "special": true
669
+ },
670
+ "83": {
671
+ "content": "<|reserved_59|>",
672
+ "lstrip": false,
673
+ "normalized": false,
674
+ "rstrip": false,
675
+ "single_word": false,
676
+ "special": true
677
+ },
678
+ "84": {
679
+ "content": "<|reserved_60|>",
680
+ "lstrip": false,
681
+ "normalized": false,
682
+ "rstrip": false,
683
+ "single_word": false,
684
+ "special": true
685
+ },
686
+ "85": {
687
+ "content": "<|reserved_61|>",
688
+ "lstrip": false,
689
+ "normalized": false,
690
+ "rstrip": false,
691
+ "single_word": false,
692
+ "special": true
693
+ },
694
+ "86": {
695
+ "content": "<|reserved_62|>",
696
+ "lstrip": false,
697
+ "normalized": false,
698
+ "rstrip": false,
699
+ "single_word": false,
700
+ "special": true
701
+ },
702
+ "87": {
703
+ "content": "<|reserved_63|>",
704
+ "lstrip": false,
705
+ "normalized": false,
706
+ "rstrip": false,
707
+ "single_word": false,
708
+ "special": true
709
+ },
710
+ "88": {
711
+ "content": "<|reserved_64|>",
712
+ "lstrip": false,
713
+ "normalized": false,
714
+ "rstrip": false,
715
+ "single_word": false,
716
+ "special": true
717
+ },
718
+ "89": {
719
+ "content": "<|reserved_65|>",
720
+ "lstrip": false,
721
+ "normalized": false,
722
+ "rstrip": false,
723
+ "single_word": false,
724
+ "special": true
725
+ },
726
+ "90": {
727
+ "content": "<|reserved_66|>",
728
+ "lstrip": false,
729
+ "normalized": false,
730
+ "rstrip": false,
731
+ "single_word": false,
732
+ "special": true
733
+ },
734
+ "91": {
735
+ "content": "<|reserved_67|>",
736
+ "lstrip": false,
737
+ "normalized": false,
738
+ "rstrip": false,
739
+ "single_word": false,
740
+ "special": true
741
+ },
742
+ "92": {
743
+ "content": "<|reserved_68|>",
744
+ "lstrip": false,
745
+ "normalized": false,
746
+ "rstrip": false,
747
+ "single_word": false,
748
+ "special": true
749
+ },
750
+ "93": {
751
+ "content": "<|reserved_69|>",
752
+ "lstrip": false,
753
+ "normalized": false,
754
+ "rstrip": false,
755
+ "single_word": false,
756
+ "special": true
757
+ },
758
+ "94": {
759
+ "content": "<|reserved_70|>",
760
+ "lstrip": false,
761
+ "normalized": false,
762
+ "rstrip": false,
763
+ "single_word": false,
764
+ "special": true
765
+ },
766
+ "95": {
767
+ "content": "<|reserved_71|>",
768
+ "lstrip": false,
769
+ "normalized": false,
770
+ "rstrip": false,
771
+ "single_word": false,
772
+ "special": true
773
+ },
774
+ "96": {
775
+ "content": "<|reserved_72|>",
776
+ "lstrip": false,
777
+ "normalized": false,
778
+ "rstrip": false,
779
+ "single_word": false,
780
+ "special": true
781
+ },
782
+ "97": {
783
+ "content": "<|reserved_73|>",
784
+ "lstrip": false,
785
+ "normalized": false,
786
+ "rstrip": false,
787
+ "single_word": false,
788
+ "special": true
789
+ },
790
+ "98": {
791
+ "content": "<|reserved_74|>",
792
+ "lstrip": false,
793
+ "normalized": false,
794
+ "rstrip": false,
795
+ "single_word": false,
796
+ "special": true
797
+ },
798
+ "99": {
799
+ "content": "<|reserved_75|>",
800
+ "lstrip": false,
801
+ "normalized": false,
802
+ "rstrip": false,
803
+ "single_word": false,
804
+ "special": true
805
+ },
806
+ "100": {
807
+ "content": "<|reserved_76|>",
808
+ "lstrip": false,
809
+ "normalized": false,
810
+ "rstrip": false,
811
+ "single_word": false,
812
+ "special": true
813
+ },
814
+ "101": {
815
+ "content": "<|reserved_77|>",
816
+ "lstrip": false,
817
+ "normalized": false,
818
+ "rstrip": false,
819
+ "single_word": false,
820
+ "special": true
821
+ },
822
+ "102": {
823
+ "content": "<|reserved_78|>",
824
+ "lstrip": false,
825
+ "normalized": false,
826
+ "rstrip": false,
827
+ "single_word": false,
828
+ "special": true
829
+ },
830
+ "103": {
831
+ "content": "<|reserved_79|>",
832
+ "lstrip": false,
833
+ "normalized": false,
834
+ "rstrip": false,
835
+ "single_word": false,
836
+ "special": true
837
+ },
838
+ "104": {
839
+ "content": "<|reserved_80|>",
840
+ "lstrip": false,
841
+ "normalized": false,
842
+ "rstrip": false,
843
+ "single_word": false,
844
+ "special": true
845
+ },
846
+ "105": {
847
+ "content": "<|reserved_81|>",
848
+ "lstrip": false,
849
+ "normalized": false,
850
+ "rstrip": false,
851
+ "single_word": false,
852
+ "special": true
853
+ },
854
+ "106": {
855
+ "content": "<|reserved_82|>",
856
+ "lstrip": false,
857
+ "normalized": false,
858
+ "rstrip": false,
859
+ "single_word": false,
860
+ "special": true
861
+ },
862
+ "107": {
863
+ "content": "<|reserved_83|>",
864
+ "lstrip": false,
865
+ "normalized": false,
866
+ "rstrip": false,
867
+ "single_word": false,
868
+ "special": true
869
+ },
870
+ "108": {
871
+ "content": "<|reserved_84|>",
872
+ "lstrip": false,
873
+ "normalized": false,
874
+ "rstrip": false,
875
+ "single_word": false,
876
+ "special": true
877
+ },
878
+ "109": {
879
+ "content": "<|reserved_85|>",
880
+ "lstrip": false,
881
+ "normalized": false,
882
+ "rstrip": false,
883
+ "single_word": false,
884
+ "special": true
885
+ },
886
+ "110": {
887
+ "content": "<|reserved_86|>",
888
+ "lstrip": false,
889
+ "normalized": false,
890
+ "rstrip": false,
891
+ "single_word": false,
892
+ "special": true
893
+ },
894
+ "111": {
895
+ "content": "<|reserved_87|>",
896
+ "lstrip": false,
897
+ "normalized": false,
898
+ "rstrip": false,
899
+ "single_word": false,
900
+ "special": true
901
+ },
902
+ "112": {
903
+ "content": "<|reserved_88|>",
904
+ "lstrip": false,
905
+ "normalized": false,
906
+ "rstrip": false,
907
+ "single_word": false,
908
+ "special": true
909
+ },
910
+ "113": {
911
+ "content": "<|reserved_89|>",
912
+ "lstrip": false,
913
+ "normalized": false,
914
+ "rstrip": false,
915
+ "single_word": false,
916
+ "special": true
917
+ },
918
+ "114": {
919
+ "content": "<|reserved_90|>",
920
+ "lstrip": false,
921
+ "normalized": false,
922
+ "rstrip": false,
923
+ "single_word": false,
924
+ "special": true
925
+ },
926
+ "115": {
927
+ "content": "<|reserved_91|>",
928
+ "lstrip": false,
929
+ "normalized": false,
930
+ "rstrip": false,
931
+ "single_word": false,
932
+ "special": true
933
+ },
934
+ "116": {
935
+ "content": "<|reserved_92|>",
936
+ "lstrip": false,
937
+ "normalized": false,
938
+ "rstrip": false,
939
+ "single_word": false,
940
+ "special": true
941
+ },
942
+ "117": {
943
+ "content": "<|reserved_93|>",
944
+ "lstrip": false,
945
+ "normalized": false,
946
+ "rstrip": false,
947
+ "single_word": false,
948
+ "special": true
949
+ },
950
+ "118": {
951
+ "content": "<|reserved_94|>",
952
+ "lstrip": false,
953
+ "normalized": false,
954
+ "rstrip": false,
955
+ "single_word": false,
956
+ "special": true
957
+ },
958
+ "119": {
959
+ "content": "<|reserved_95|>",
960
+ "lstrip": false,
961
+ "normalized": false,
962
+ "rstrip": false,
963
+ "single_word": false,
964
+ "special": true
965
+ },
966
+ "120": {
967
+ "content": "<|reserved_96|>",
968
+ "lstrip": false,
969
+ "normalized": false,
970
+ "rstrip": false,
971
+ "single_word": false,
972
+ "special": true
973
+ },
974
+ "121": {
975
+ "content": "<|reserved_97|>",
976
+ "lstrip": false,
977
+ "normalized": false,
978
+ "rstrip": false,
979
+ "single_word": false,
980
+ "special": true
981
+ },
982
+ "122": {
983
+ "content": "<|reserved_98|>",
984
+ "lstrip": false,
985
+ "normalized": false,
986
+ "rstrip": false,
987
+ "single_word": false,
988
+ "special": true
989
+ },
990
+ "123": {
991
+ "content": "<|reserved_99|>",
992
+ "lstrip": false,
993
+ "normalized": false,
994
+ "rstrip": false,
995
+ "single_word": false,
996
+ "special": true
997
+ },
998
+ "124": {
999
+ "content": "<|reserved_100|>",
1000
+ "lstrip": false,
1001
+ "normalized": false,
1002
+ "rstrip": false,
1003
+ "single_word": false,
1004
+ "special": true
1005
+ },
1006
+ "125": {
1007
+ "content": "<|reserved_101|>",
1008
+ "lstrip": false,
1009
+ "normalized": false,
1010
+ "rstrip": false,
1011
+ "single_word": false,
1012
+ "special": true
1013
+ },
1014
+ "126": {
1015
+ "content": "<|reserved_102|>",
1016
+ "lstrip": false,
1017
+ "normalized": false,
1018
+ "rstrip": false,
1019
+ "single_word": false,
1020
+ "special": true
1021
+ },
1022
+ "127": {
1023
+ "content": "<|reserved_103|>",
1024
+ "lstrip": false,
1025
+ "normalized": false,
1026
+ "rstrip": false,
1027
+ "single_word": false,
1028
+ "special": true
1029
+ }
1030
+ },
1031
+ "additional_special_tokens": [
1032
+ "<|im_start|>",
1033
+ "<|im_end|>",
1034
+ "<|system|>",
1035
+ "<|user|>",
1036
+ "<|assistant|>",
1037
+ "<|tool|>",
1038
+ "<think>",
1039
+ "</think>",
1040
+ "<tool_call>",
1041
+ "</tool_call>",
1042
+ "<tool_response>",
1043
+ "</tool_response>",
1044
+ "<image_start>",
1045
+ "<image_pad>",
1046
+ "<image_end>",
1047
+ "<audio_start>",
1048
+ "<audio_pad>",
1049
+ "<audio_end>",
1050
+ "<speech_bos>",
1051
+ "<speech_eos>",
1052
+ "<speech_pad>"
1053
+ ],
1054
+ "bos_token": "<|bos|>",
1055
+ "clean_up_tokenization_spaces": false,
1056
+ "eos_token": "<|eos|>",
1057
+ "extra_special_tokens": {},
1058
+ "legacy": true,
1059
+ "model_max_length": 4096,
1060
+ "pad_token": "<|pad|>",
1061
+ "spaces_between_special_tokens": false,
1062
+ "tokenizer_class": "PreTrainedTokenizerFast",
1063
+ "unk_token": "<|unk|>"
1064
+ }