add: model lfs

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9eef7010bd80b2d736ce95af88b32b2af738406247c7fdc9f6f2b84922c61fc
+size 16031197344

modelling_llada.py

@@ -0,0 +1,457 @@
+# adapted from https://huggingface.co/GSAI-ML/LLaDA-8B-Base/blob/main/modeling_llada.py
+
+from typing import (
+    Optional,
+    Tuple,
+)
+
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import einsum
+
+class RMSLayerNorm(nn.Module):
+    """
+    RMS layer norm, a simplified :class:`LayerNorm` implementation
+    """
+
+    def __init__(
+        self,
+        d_model: int,
+        eps: float = 1e-5,
+        device: torch.device = "cuda",
+    ):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(d_model, device=device))
+
+        nn.init.ones_(self.weight)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        og_dtype = x.dtype
+        x = x.to(torch.float32)
+        variance = x.pow(2).mean(-1, keepdim=True)
+        x = x * torch.rsqrt(variance + self.eps)
+        x = x.to(og_dtype)
+
+        return self.weight * x
+
+class RotaryEmbedding(nn.Module):
+    def __init__(
+            self,
+            rope_theta:float, 
+            d_model: int,
+            n_heads: int,
+            max_sequence_length: int, 
+            device: torch.device
+        ):
+        super().__init__()
+        self.rope_theta = rope_theta
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.get_rotary_embedding(max_sequence_length, device)
+
+    def get_rotary_embedding(self, seq_len: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]:
+
+        dim = self.d_model // self.n_heads
+        inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float) / dim))
+        seq = torch.arange(seq_len, device=device, dtype=torch.float)
+        freqs = einsum("i , j -> i j", seq, inv_freq)
+        positions = torch.cat((freqs, freqs), dim=-1)
+        pos_sin, pos_cos = positions.sin()[None, None, :, :], positions.cos()[None, None, :, :]
+
+        return pos_sin, pos_cos
+
+    def rotate_half(self, x: torch.Tensor) -> torch.Tensor:
+        B, nh, T, hs = x.size()
+        x = x.view(B, nh, T, 2, hs // 2)
+        x1, x2 = x.unbind(dim=-2)
+
+        return torch.cat((-x2, x1), dim=-1)
+
+    def apply_rotary_pos_emb(self, pos_sin: torch.Tensor, pos_cos: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
+        return ((t * pos_cos) + (self.rotate_half(t) * pos_sin)).to(t.dtype)
+
+    def forward(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        query_len, key_len = q.shape[-2], k.shape[-2]  # could be different if layer_past not None
+        pos_sin, pos_cos = self.get_rotary_embedding(key_len, q.device)
+        pos_sin = pos_sin.type_as(q)
+        pos_cos = pos_cos.type_as(q)
+        q = self.apply_rotary_pos_emb(
+            pos_sin[:, :, key_len - query_len : key_len, :],
+            pos_cos[:, :, key_len - query_len : key_len, :],
+            q,
+        )
+        k = self.apply_rotary_pos_emb(pos_sin, pos_cos, k)
+
+        return q.type_as(q), k.type_as(k)
+
+class SwiGLU(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+    @property
+    def output_multiplier(self) -> float:
+        return 0.5  
+
+
+class SiLU(nn.SiLU):
+    @property
+    def output_multiplier(self) -> float:
+        return 1.0
+
+
+class LLaDALlamaBlock(nn.Module):
+    """
+    This is a transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
+    (plus another skip connection). This block is similar to `LLaDASequentialBlock`
+    but some operations have slightly different implementations to imitate the
+    behavior of Llama.
+    """
+
+    def __init__(
+            self, 
+            layer_id: int, 
+            mlp_ratio: int,
+            d_model: int,
+            n_heads: int, 
+            rope_theta: float,
+            max_sequence_length: int,
+            mlp_hidden_size: int,
+            device: torch.device,
+        ):
+        super().__init__()
+        self.layer_id = layer_id
+        self.hidden_size = (
+            mlp_hidden_size  if mlp_hidden_size is not None else mlp_ratio * d_model
+        )
+        assert d_model % n_heads == 0
+
+        self.n_heads = n_heads
+
+        # Activation function.
+        self.act = SiLU()
+        assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
+
+        # Attention output projection.
+        self.attn_out = nn.Linear(
+            d_model, d_model, bias=False, device=device
+        )
+
+        # Feed-forward output projection.
+        self.ff_out = nn.Linear(
+            int(self.act.output_multiplier * self.hidden_size),
+            d_model,
+            bias=False,
+            device=device,
+        )
+        self.ff_out._is_residual = True
+
+        # Rotary embeddings.
+        self.rotary_emb = RotaryEmbedding(rope_theta=rope_theta, d_model=d_model, n_heads=n_heads, max_sequence_length=max_sequence_length, device=device)
+
+        # Layer norms.
+        self.attn_norm = RMSLayerNorm(d_model=d_model, device=device)
+        self.ff_norm = RMSLayerNorm(d_model=d_model, device=device)
+        
+        # Attention input projection. Projects x -> (q, k, v)
+        q_proj_out_dim = d_model
+        k_proj_out_dim = d_model
+        v_proj_out_dim = d_model
+        self.q_proj = nn.Linear(
+            d_model, q_proj_out_dim, bias=False, device=device,
+        )
+        self.k_proj = nn.Linear(
+            d_model, k_proj_out_dim, bias=False, device=device,
+        )
+        self.v_proj = nn.Linear(
+            d_model, v_proj_out_dim, bias=False, device=device,
+        )
+        
+        # Feed-forward input projection.
+        self.ff_proj = nn.Linear(
+            d_model, self.hidden_size, bias=False, device=device
+        )
+        # new add
+        self.up_proj = nn.Linear(
+            d_model, self.hidden_size, bias=False, device=device,
+        )
+
+    def attention(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        B, T, C = q.size()  # batch size, sequence length, d_model
+
+        # Move head forward to be next to the batch dim.
+        # shape: (B, nh, T, hs)
+        q = q.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
+        # shape: (B, n_kv_h, T, hs)
+        k = k.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
+        # shape: (B, n_kv_h, T, hs)
+        v = v.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
+
+        q, k = self.rotary_emb(q, k)
+
+        # Get the attention scores.
+        # shape: (B, nh, T, hs)
+        att = F.scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            attn_mask=None,
+            dropout_p=0.0,
+            is_causal=False,
+        )
+
+        # Re-assemble all head outputs side-by-side.
+        att = att.transpose(1, 2).contiguous().view(B, T, C)
+
+        # Apply output projection.
+        return self.attn_out(att)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        x_normed = self.attn_norm(x)
+        q = self.q_proj(x_normed)
+        k = self.k_proj(x_normed)
+        v = self.v_proj(x_normed)
+
+        att = self.attention(q, k, v)
+
+        # Add attention scores.
+        # shape: (B, T, C)
+        x = x + att
+
+        # Add feed-forward projection.
+        # shape: (batch_size, seq_len, d_model)
+        og_x = x
+        x = self.ff_norm(x)
+        x, x_up = self.ff_proj(x), self.up_proj(x) # new add
+        
+        x = self.act(x)
+        x = x * x_up # new add
+        x = self.ff_out(x)
+        x = og_x + x
+
+        return x
+
+
+class LLaDASequentialBlock(nn.Module):
+    """
+    This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
+    (plus another skip connection).
+    """
+
+    def __init__(
+            self, 
+            layer_id: int, 
+            mlp_ratio: int,
+            d_model: int,
+            n_heads: int, 
+            rope_theta: float,
+            max_sequence_length: int,
+            mlp_hidden_size: int,
+            device: torch.device,
+        ):
+        super().__init__()
+        self.layer_id = layer_id
+        self.hidden_size = (
+            mlp_hidden_size if mlp_hidden_size is not None else mlp_ratio * d_model
+        )
+        assert d_model % n_heads == 0
+
+        self.n_heads = n_heads
+
+        # Activation function.
+        self.act = SwiGLU()
+        assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
+
+        # Attention output projection.
+        self.attn_out = nn.Linear(
+            d_model, d_model, bias=False, device=device
+        )
+
+        # Feed-forward output projection.
+        self.ff_out = nn.Linear(
+            int(self.act.output_multiplier * self.hidden_size),
+            d_model,
+            bias=False,
+            device=device,
+        )
+        self.ff_out._is_residual = True
+
+        # Rotary embeddings.
+        self.rotary_emb = RotaryEmbedding(rope_theta=rope_theta, d_model=d_model, n_heads=n_heads, max_sequence_length=max_sequence_length, device=device)
+
+        # Layer norms.
+        self.attn_norm = RMSLayerNorm(d_model=d_model, device=device)
+        self.ff_norm = RMSLayerNorm(d_model=d_model, device=device)
+        
+        # Attention input projection. Projects x -> (q, k, v)
+        self.fused_dims = (
+            d_model,
+            d_model,
+            d_model,
+        )
+        self.att_proj = nn.Linear(
+            d_model, sum(self.fused_dims), bias=False, device=device
+        )
+        
+        # Feed-forward input projection.
+        self.ff_proj = nn.Linear(
+            d_model, self.hidden_size, bias=False, device=device
+        )
+
+    def attention(
+        self,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        B, T, C = q.size()  # batch size, sequence length, d_model
+
+        # Move head forward to be next to the batch dim.
+        # shape: (B, nh, T, hs)
+        q = q.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
+        # shape: (B, n_kv_h, T, hs)
+        k = k.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
+        # shape: (B, n_kv_h, T, hs)
+        v = v.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
+
+        q, k = self.rotary_emb(q, k)
+
+        # Get the attention scores.
+        # shape: (B, nh, T, hs)
+        att = F.scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            attn_mask=None,
+            dropout_p=0.0,
+            is_causal=False,
+        )
+
+        # Re-assemble all head outputs side-by-side.
+        att = att.transpose(1, 2).contiguous().view(B, T, C)
+
+        # Apply output projection.
+        return self.attn_out(att)
+
+    def forward(
+        self,
+        x: torch.Tensor,
+    ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
+        q, k, v = self.att_proj(self.attn_norm(x)).split(self.fused_dims, dim=-1)
+
+        att = self.attention(q, k, v)
+
+        # Add attention scores.
+        # shape: (B, T, C)
+        x = x + att
+
+        # Add feed-forward projection.
+        # shape: (batch_size, seq_len, d_model)
+        og_x = x
+        x = self.ff_norm(x)
+        x = self.ff_proj(x)
+
+        x = self.act(x)
+        x = self.ff_out(x)
+        x = og_x + x
+
+        return x
+
+class LLaDAModel(nn.Module):
+    def __init__(
+            self, 
+            mlp_ratio: int,
+            d_model: int,
+            n_heads: int, 
+            rope_theta: float,
+            max_sequence_length: int,
+            vocab_size: int,
+            n_layers: int,
+            mlp_hidden_size: int,
+            device: torch.device,
+        ):
+        super().__init__()
+        self.transformer = nn.ModuleDict(
+            dict(
+                wte=nn.Embedding(
+                    vocab_size, d_model, device=device
+                ),
+                ln_f=RMSLayerNorm(d_model=d_model, device=device),
+            )
+        )
+
+        blocks = [
+            LLaDALlamaBlock(
+                layer_id=i, 
+                mlp_ratio=mlp_ratio,
+                d_model=d_model,
+                n_heads=n_heads, 
+                rope_theta=rope_theta,
+                max_sequence_length=max_sequence_length,
+                mlp_hidden_size=mlp_hidden_size,
+                device=device,
+            ) 
+            for i in range(n_layers)
+        ]
+        self.transformer.update({"blocks": nn.ModuleList(blocks)})
+
+        self.transformer.update(
+            {
+                "ff_out": nn.Linear(
+                    d_model,
+                    vocab_size,
+                    bias=False,
+                    device=device,
+                )
+            }
+        )
+
+    @property
+    def device(self) -> torch.device:
+        device: torch.device = self.transformer.wte.weight.device  # type: ignore
+        return device
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        last_logits_only: bool = False,
+    ) -> torch.Tensor:
+        """
+        :param input_ids: A tensor of shape `(batch_size, seq_len)`.
+        :param last_logits_only: If `True`, only compute the logits for the last token of each sequence.
+            This can speed up decoding when you only care about the next token.
+        """
+
+        # Get embeddings of input.
+        # shape: (batch_size, seq_len, d_model)
+        x = self.transformer.wte(input_ids)
+
+        for block_idx, block in enumerate(self.transformer.blocks):
+            x = block(x)
+
+        if last_logits_only:
+            # shape: (batch_size, 1, d_model)
+            x = x[:, -1, :].unsqueeze(1)
+
+        # Apply final layer norm.
+        # shape: (batch_size, seq_len or 1, d_model)
+        x = self.transformer.ln_f(x)  # type: ignore
+
+        logits = self.transformer.ff_out(x)  # type: ignore
+
+        return logits

special_tokens.json

@@ -0,0 +1,271 @@
+{
+  "<|startoftext|>": 126080,
+  "<|endoftext|>": 126081,
+  "[CLS]": 126082,
+  "[gMASK]": 126083,
+  "<|reserved_token_0|>": 126084,
+  "<|reserved_token_1|>": 126085,
+  "<|reserved_token_2|>": 126086,
+  "<|reserved_token_3|>": 126087,
+  "<|reserved_token_4|>": 126088,
+  "<|reserved_token_5|>": 126089,
+  "<|reserved_token_6|>": 126090,
+  "<|reserved_token_7|>": 126091,
+  "<|reserved_token_8|>": 126092,
+  "<|reserved_token_9|>": 126093,
+  "<|reserved_token_10|>": 126094,
+  "<|reserved_token_11|>": 126095,
+  "<|reserved_token_12|>": 126096,
+  "<|reserved_token_13|>": 126097,
+  "<|reserved_token_14|>": 126098,
+  "<|reserved_token_15|>": 126099,
+  "<|reserved_token_16|>": 126100,
+  "<|reserved_token_17|>": 126101,
+  "<|reserved_token_18|>": 126102,
+  "<|reserved_token_19|>": 126103,
+  "<|reserved_token_20|>": 126104,
+  "<|reserved_token_21|>": 126105,
+  "<|reserved_token_22|>": 126106,
+  "<|reserved_token_23|>": 126107,
+  "<|reserved_token_24|>": 126108,
+  "<|reserved_token_25|>": 126109,
+  "<|reserved_token_26|>": 126110,
+  "<|reserved_token_27|>": 126111,
+  "<|reserved_token_28|>": 126112,
+  "<|reserved_token_29|>": 126113,
+  "<|reserved_token_30|>": 126114,
+  "<|reserved_token_31|>": 126115,
+  "<|reserved_token_32|>": 126116,
+  "<|reserved_token_33|>": 126117,
+  "<|reserved_token_34|>": 126118,
+  "<|reserved_token_35|>": 126119,
+  "<|reserved_token_36|>": 126120,
+  "<|reserved_token_37|>": 126121,
+  "<|reserved_token_38|>": 126122,
+  "<|reserved_token_39|>": 126123,
+  "<|reserved_token_40|>": 126124,
+  "<|reserved_token_41|>": 126125,
+  "<|reserved_token_42|>": 126126,
+  "<|reserved_token_43|>": 126127,
+  "<|reserved_token_44|>": 126128,
+  "<|reserved_token_45|>": 126129,
+  "<|reserved_token_46|>": 126130,
+  "<|reserved_token_47|>": 126131,
+  "<|reserved_token_48|>": 126132,
+  "<|reserved_token_49|>": 126133,
+  "<|reserved_token_50|>": 126134,
+  "<|reserved_token_51|>": 126135,
+  "<|reserved_token_52|>": 126136,
+  "<|reserved_token_53|>": 126137,
+  "<|reserved_token_54|>": 126138,
+  "<|reserved_token_55|>": 126139,
+  "<|reserved_token_56|>": 126140,
+  "<|reserved_token_57|>": 126141,
+  "<|reserved_token_58|>": 126142,
+  "<|reserved_token_59|>": 126143,
+  "<|reserved_token_60|>": 126144,
+  "<|reserved_token_61|>": 126145,
+  "<|reserved_token_62|>": 126146,
+  "<|reserved_token_63|>": 126147,
+  "<|reserved_token_64|>": 126148,
+  "<|reserved_token_65|>": 126149,
+  "<|reserved_token_66|>": 126150,
+  "<|reserved_token_67|>": 126151,
+  "<|reserved_token_68|>": 126152,
+  "<|reserved_token_69|>": 126153,
+  "<|reserved_token_70|>": 126154,
+  "<|reserved_token_71|>": 126155,
+  "<|reserved_token_72|>": 126156,
+  "<|reserved_token_73|>": 126157,
+  "<|reserved_token_74|>": 126158,
+  "<|reserved_token_75|>": 126159,
+  "<|reserved_token_76|>": 126160,
+  "<|reserved_token_77|>": 126161,
+  "<|reserved_token_78|>": 126162,
+  "<|reserved_token_79|>": 126163,
+  "<|reserved_token_80|>": 126164,
+  "<|reserved_token_81|>": 126165,
+  "<|reserved_token_82|>": 126166,
+  "<|reserved_token_83|>": 126167,
+  "<|reserved_token_84|>": 126168,
+  "<|reserved_token_85|>": 126169,
+  "<|reserved_token_86|>": 126170,
+  "<|reserved_token_87|>": 126171,
+  "<|reserved_token_88|>": 126172,
+  "<|reserved_token_89|>": 126173,
+  "<|reserved_token_90|>": 126174,
+  "<|reserved_token_91|>": 126175,
+  "<|reserved_token_92|>": 126176,
+  "<|reserved_token_93|>": 126177,
+  "<|reserved_token_94|>": 126178,
+  "<|reserved_token_95|>": 126179,
+  "<|reserved_token_96|>": 126180,
+  "<|reserved_token_97|>": 126181,
+  "<|reserved_token_98|>": 126182,
+  "<|reserved_token_99|>": 126183,
+  "<|reserved_token_100|>": 126184,
+  "<|reserved_token_101|>": 126185,
+  "<|reserved_token_102|>": 126186,
+  "<|reserved_token_103|>": 126187,
+  "<|reserved_token_104|>": 126188,
+  "<|reserved_token_105|>": 126189,
+  "<|reserved_token_106|>": 126190,
+  "<|reserved_token_107|>": 126191,
+  "<|reserved_token_108|>": 126192,
+  "<|reserved_token_109|>": 126193,
+  "<|reserved_token_110|>": 126194,
+  "<|reserved_token_111|>": 126195,
+  "<|reserved_token_112|>": 126196,
+  "<|reserved_token_113|>": 126197,
+  "<|reserved_token_114|>": 126198,
+  "<|reserved_token_115|>": 126199,
+  "<|reserved_token_116|>": 126200,
+  "<|reserved_token_117|>": 126201,
+  "<|reserved_token_118|>": 126202,
+  "<|reserved_token_119|>": 126203,
+  "<|reserved_token_120|>": 126204,
+  "<|reserved_token_121|>": 126205,
+  "<|reserved_token_122|>": 126206,
+  "<|reserved_token_123|>": 126207,
+  "<|reserved_token_124|>": 126208,
+  "<|reserved_token_125|>": 126209,
+  "<|reserved_token_126|>": 126210,
+  "<|reserved_token_127|>": 126211,
+  "<|reserved_token_128|>": 126212,
+  "<|reserved_token_129|>": 126213,
+  "<|reserved_token_130|>": 126214,
+  "<|reserved_token_131|>": 126215,
+  "<|reserved_token_132|>": 126216,
+  "<|reserved_token_133|>": 126217,
+  "<|reserved_token_134|>": 126218,
+  "<|reserved_token_135|>": 126219,
+  "<|reserved_token_136|>": 126220,
+  "<|reserved_token_137|>": 126221,
+  "<|reserved_token_138|>": 126222,
+  "<|reserved_token_139|>": 126223,
+  "<|reserved_token_140|>": 126224,
+  "<|reserved_token_141|>": 126225,
+  "<|reserved_token_142|>": 126226,
+  "<|reserved_token_143|>": 126227,
+  "<|reserved_token_144|>": 126228,
+  "<|reserved_token_145|>": 126229,
+  "<|reserved_token_146|>": 126230,
+  "<|reserved_token_147|>": 126231,
+  "<|reserved_token_148|>": 126232,
+  "<|reserved_token_149|>": 126233,
+  "<|reserved_token_150|>": 126234,
+  "<|reserved_token_151|>": 126235,
+  "<|reserved_token_152|>": 126236,
+  "<|reserved_token_153|>": 126237,
+  "<|reserved_token_154|>": 126238,
+  "<|reserved_token_155|>": 126239,
+  "<|reserved_token_156|>": 126240,
+  "<|reserved_token_157|>": 126241,
+  "<|reserved_token_158|>": 126242,
+  "<|reserved_token_159|>": 126243,
+  "<|reserved_token_160|>": 126244,
+  "<|reserved_token_161|>": 126245,
+  "<|reserved_token_162|>": 126246,
+  "<|reserved_token_163|>": 126247,
+  "<|reserved_token_164|>": 126248,
+  "<|reserved_token_165|>": 126249,
+  "<|reserved_token_166|>": 126250,
+  "<|reserved_token_167|>": 126251,
+  "<|reserved_token_168|>": 126252,
+  "<|reserved_token_169|>": 126253,
+  "<|reserved_token_170|>": 126254,
+  "<|reserved_token_171|>": 126255,
+  "<|reserved_token_172|>": 126256,
+  "<|reserved_token_173|>": 126257,
+  "<|reserved_token_174|>": 126258,
+  "<|reserved_token_175|>": 126259,
+  "<|reserved_token_176|>": 126260,
+  "<|reserved_token_177|>": 126261,
+  "<|reserved_token_178|>": 126262,
+  "<|reserved_token_179|>": 126263,
+  "<|reserved_token_180|>": 126264,
+  "<|reserved_token_181|>": 126265,
+  "<|reserved_token_182|>": 126266,
+  "<|reserved_token_183|>": 126267,
+  "<|reserved_token_184|>": 126268,
+  "<|reserved_token_185|>": 126269,
+  "<|reserved_token_186|>": 126270,
+  "<|reserved_token_187|>": 126271,
+  "<|reserved_token_188|>": 126272,
+  "<|reserved_token_189|>": 126273,
+  "<|reserved_token_190|>": 126274,
+  "<|reserved_token_191|>": 126275,
+  "<|reserved_token_192|>": 126276,
+  "<|reserved_token_193|>": 126277,
+  "<|reserved_token_194|>": 126278,
+  "<|reserved_token_195|>": 126279,
+  "<|reserved_token_196|>": 126280,
+  "<|reserved_token_197|>": 126281,
+  "<|reserved_token_198|>": 126282,
+  "<|reserved_token_199|>": 126283,
+  "<|reserved_token_200|>": 126284,
+  "<|reserved_token_201|>": 126285,
+  "<|reserved_token_202|>": 126286,
+  "<|reserved_token_203|>": 126287,
+  "<|reserved_token_204|>": 126288,
+  "<|reserved_token_205|>": 126289,
+  "<|reserved_token_206|>": 126290,
+  "<|reserved_token_207|>": 126291,
+  "<|reserved_token_208|>": 126292,
+  "<|reserved_token_209|>": 126293,
+  "<|reserved_token_210|>": 126294,
+  "<|reserved_token_211|>": 126295,
+  "<|reserved_token_212|>": 126296,
+  "<|reserved_token_213|>": 126297,
+  "<|reserved_token_214|>": 126298,
+  "<|reserved_token_215|>": 126299,
+  "<|reserved_token_216|>": 126300,
+  "<|reserved_token_217|>": 126301,
+  "<|reserved_token_218|>": 126302,
+  "<|reserved_token_219|>": 126303,
+  "<|reserved_token_220|>": 126304,
+  "<|reserved_token_221|>": 126305,
+  "<|reserved_token_222|>": 126306,
+  "<|reserved_token_223|>": 126307,
+  "<|reserved_token_224|>": 126308,
+  "<|reserved_token_225|>": 126309,
+  "<|reserved_token_226|>": 126310,
+  "<|reserved_token_227|>": 126311,
+  "<|reserved_token_228|>": 126312,
+  "<|reserved_token_229|>": 126313,
+  "<|reserved_token_230|>": 126314,
+  "<|reserved_token_231|>": 126315,
+  "<|reserved_token_232|>": 126316,
+  "<|reserved_token_233|>": 126317,
+  "<|reserved_token_234|>": 126318,
+  "<|reserved_token_235|>": 126319,
+  "<|reserved_token_236|>": 126320,
+  "<|reserved_token_237|>": 126321,
+  "<|reserved_token_238|>": 126322,
+  "<|reserved_token_239|>": 126323,
+  "<|reserved_token_240|>": 126324,
+  "<|reserved_token_241|>": 126325,
+  "<|reserved_token_242|>": 126326,
+  "<|reserved_token_243|>": 126327,
+  "<|reserved_token_244|>": 126328,
+  "<|reserved_token_245|>": 126329,
+  "<|reserved_token_246|>": 126330,
+  "<|reserved_token_247|>": 126331,
+  "<|reserved_token_248|>": 126332,
+  "<|reserved_token_249|>": 126333,
+  "<|reserved_token_250|>": 126334,
+  "<|reserved_token_251|>": 126335,
+  "<|mdm_mask|>": 126336,
+  "<|reserved_token_253|>": 126337,
+  "<|reserved_token_254|>": 126338,
+  "<|reserved_token_255|>": 126339,
+  "<role>": 126340,
+  "</role>": 126341,
+  "<|arithmetic_start|>": 126342,
+  "<|arithmetic_end|>": 126343,
+  "<|number_start|>": 126344,
+  "<|number_end|>": 126345,
+  "<|start_header_id|>": 126346,
+  "<|end_header_id|>": 126347,
+  "<|eot_id|>": 126348
+}