Push to HF

config.json

@@ -0,0 +1,38 @@
+{
+  "model_type": "basemini",
+  "model_name": "Basemini",
+  "architectures": [
+    "TinyGPTForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_tiny_gpt.TinyGPTConfig",
+    "AutoModel": "modeling_tiny_gpt.TinyGPTModel",
+    "AutoModelForCausalLM": "modeling_tiny_gpt.TinyGPTForCausalLM"
+  },
+  "vocab_size": 466,
+  "ctx_len": 128,
+  "max_position_embeddings": 128,
+  "n_layer": 4,
+  "num_hidden_layers": 4,
+  "n_head": 4,
+  "num_attention_heads": 4,
+  "n_embd": 384,
+  "hidden_size": 384,
+  "dropout": 0.0,
+  "attention_backend": "sage",
+  "available_attention_backends": [
+    "sage",
+    "torch",
+    "flash2",
+    "flash3"
+  ],
+  "trained_attention_backend": "torch",
+  "torch_fallback": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "custom",
+  "pad_token_id": 0,
+  "bos_token_id": 5,
+  "eos_token_id": 6,
+  "sep_token_id": 2,
+  "unk_token_id": 1
+}

configuration_tiny_gpt.py

@@ -0,0 +1,51 @@
+from transformers import PretrainedConfig
+
+
+class TinyGPTConfig(PretrainedConfig):
+    model_type = "basemini"
+
+    def __init__(
+        self,
+        vocab_size=32768,
+        ctx_len=512,
+        n_layer=4,
+        n_head=4,
+        n_embd=384,
+        dropout=0.0,
+        attention_backend="torch",
+        torch_fallback=False,
+        pad_token_id=None,
+        bos_token_id=None,
+        eos_token_id=None,
+        sep_token_id=None,
+        unk_token_id=None,
+        **kwargs,
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            sep_token_id=sep_token_id,
+            unk_token_id=unk_token_id,
+            **kwargs,
+        )
+
+        if attention_backend not in ("sage", "torch", "flash2", "flash3"):
+            raise ValueError("attention_backend must be sage, torch, flash2 or flash3")
+
+        self.vocab_size = int(vocab_size)
+        self.ctx_len = int(ctx_len)
+        self.max_position_embeddings = int(ctx_len)
+
+        self.n_layer = int(n_layer)
+        self.n_head = int(n_head)
+        self.n_embd = int(n_embd)
+
+        self.num_hidden_layers = int(n_layer)
+        self.num_attention_heads = int(n_head)
+        self.hidden_size = int(n_embd)
+
+        self.dropout = float(dropout)
+        self.attention_backend = str(attention_backend)
+        self.available_attention_backends = ["sage", "torch", "flash2", "flash3"]
+        self.torch_fallback = bool(torch_fallback)

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "pad_token_id": 0,
+  "bos_token_id": 5,
+  "eos_token_id": 6,
+  "sep_token_id": 2
+}

merges.txt

@@ -0,0 +1,204 @@
+#version: 0.2
+e s
+e n
+t h
+th e
+a t
+o r
+d e
+at es
+u t
+c o
+r a
+d i
+Ġ p
+Ġ t
+n s
+en t
+s t
+Ġp r
+Ġ a
+i o
+t s
+Ġ de
+Ġde co
+l i
+w or
+wor k
+Ġ f
+n e
+Ġpr o
+Ġ g
+d es
+Ġ e
+e r
+p ut
+a l
+de r
+c es
+f or
+ra ns
+Ġt rans
+for m
+Ġtrans form
+Ġ s
+Ġ o
+z es
+io ns
+Ġ en
+Ġen co
+r e
+Ġdeco des
+a b
+b ut
+b ab
+i li
+i but
+r ibut
+t y
+Ġ di
+st ribut
+Ġpro bab
+ili ty
+Ġdi stribut
+Ġprobab ility
+Ġdistribut ions
+d d
+h i
+Ġ st
+Ġ hi
+dd en
+Ġst ates
+Ġhi dden
+en er
+Ġg ener
+Ġgener ates
+e work
+m ework
+ra mework
+Ġf ramework
+a s
+c l
+f i
+i fi
+s ifi
+Ġ cl
+as sifi
+Ġcl assifi
+Ġclassifi es
+b e
+d di
+g s
+m be
+n gs
+Ġe mbe
+ddi ngs
+Ġembe ddings
+t work
+Ġ ne
+Ġne twork
+e li
+i p
+Ġp ip
+eli ne
+Ġpip eline
+Ġdeco der
+c t
+e at
+e ct
+u re
+v ect
+Ġ vect
+or s
+Ġf eat
+Ġvect ors
+Ġfeat ure
+a n
+g ent
+Ġa gent
+m o
+Ġ mo
+de l
+Ġmo del
+i n
+k en
+o ken
+Ġ in
+Ġt oken
+Ġin put
+Ġtoken s
+g i
+l o
+Ġ lo
+ut put
+Ġo utput
+gi ts
+Ġlo gits
+u ates
+v al
+Ġe val
+Ġtransform s
+Ġeval uates
+s es
+Ġpro ces
+Ġproces ses
+d ates
+p dates
+u pdates
+Ġ updates
+ra di
+Ġg radi
+Ġgradi ent
+n al
+y zes
+Ġa nal
+Ġanal yzes
+e q
+e x
+u en
+Ġt ex
+Ġs eq
+uen ces
+Ġtex t
+Ġseq uences
+e m
+y st
+Ġs yst
+Ġsyst em
+Ġtransform er
+Ġenco des
+i m
+i zes
+p t
+Ġo pt
+im izes
+Ġopt imizes
+Ġenco der
+e i
+g h
+t ent
+w ei
+Ġ at
+Ġ wei
+io n
+gh ts
+tent ion
+Ġat tention
+Ġwei ghts
+g or
+i th
+l gor
+Ġa lgor
+ith m
+Ġalgor ithm
+c ts
+e di
+Ġpr edi
+Ġpredi cts
+p r
+Ġ re
+es ent
+at ions
+pr esent
+Ġre present
+Ġrepresent ations

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:901338c111b568e1be77b219675718d36ab84ef98f878ea7f7f4488ba0a301d4
+size 14987368

modeling_tiny_gpt.py

@@ -0,0 +1,235 @@
+import importlib
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel, GenerationMixin
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from .configuration_tiny_gpt import TinyGPTConfig
+_FLASH2_KERNEL = None
+_FLASH3_KERNEL = None
+def _get_flash2_kernel():
+    global _FLASH2_KERNEL
+    if _FLASH2_KERNEL is None:
+        kernels = importlib.import_module("kernels")
+        _FLASH2_KERNEL = kernels.get_kernel("kernels-community/flash-attn2", version=1)
+    return _FLASH2_KERNEL
+def _get_flash3_kernel():
+    global _FLASH3_KERNEL
+    if _FLASH3_KERNEL is None:
+        kernels = importlib.import_module("kernels")
+        _FLASH3_KERNEL = kernels.get_kernel("kernels-community/flash-attn3", version=1)
+    return _FLASH3_KERNEL
+def _get_sageattn():
+    module = importlib.import_module("sageattention")
+    return module.sageattn
+class CausalSelfAttention(nn.Module):
+    def __init__(self, config: TinyGPTConfig):
+        super().__init__()
+        if config.n_embd % config.n_head != 0:
+            raise ValueError("n_embd must be divisible by n_head")
+        self.n_head = int(config.n_head)
+        self.head_dim = int(config.n_embd // config.n_head)
+        self.attention_backend = str(getattr(config, "attention_backend", "torch"))
+        self.torch_fallback = bool(getattr(config, "torch_fallback", False))
+        self.dropout_p = float(config.dropout)
+        if self.attention_backend not in ("sage", "torch", "flash2", "flash3"):
+            raise ValueError("attention_backend must be sage, torch, flash2 or flash3")
+        if self.attention_backend == "sage" and self.head_dim not in (64, 96, 128):
+            raise ValueError(f"SageAttention requires head_dim in [64, 96, 128], got {self.head_dim}")
+        if self.attention_backend == "sage" and self.dropout_p != 0.0:
+            raise ValueError("SageAttention requires dropout=0.0")
+        if self.attention_backend == "flash3" and self.dropout_p != 0.0:
+            raise ValueError("FlashAttention3 requires dropout=0.0")
+        if self.attention_backend in ("flash2", "flash3") and self.head_dim % 8 != 0:
+            raise ValueError(f"FlashAttention requires head_dim multiple of 8, got {self.head_dim}")
+        self.qkv = nn.Linear(config.n_embd, 3 * config.n_embd, bias=False)
+        self.proj = nn.Linear(config.n_embd, config.n_embd, bias=False)
+        self.dropout = nn.Dropout(config.dropout)
+        mask = torch.tril(torch.ones(config.ctx_len, config.ctx_len, dtype=torch.bool))
+        self.register_buffer("mask", mask.view(1, 1, config.ctx_len, config.ctx_len), persistent=False)
+        self.sageattn = None
+        self.flash_kernel = None
+        if self.attention_backend == "sage":
+            try:
+                self.sageattn = _get_sageattn()
+            except Exception:
+                if self.torch_fallback:
+                    self.attention_backend = "torch"
+                else:
+                    raise
+        if self.attention_backend == "flash2":
+            try:
+                self.flash_kernel = _get_flash2_kernel()
+            except Exception:
+                if self.torch_fallback:
+                    self.attention_backend = "torch"
+                else:
+                    raise
+        if self.attention_backend == "flash3":
+            try:
+                self.flash_kernel = _get_flash3_kernel()
+            except Exception:
+                if self.torch_fallback:
+                    self.attention_backend = "torch"
+                else:
+                    raise
+    def _torch_attention(self, q, k, v, t):
+        scores = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
+        scores = scores.masked_fill(self.mask[:, :, :t, :t] == 0, float("-inf"))
+        att = F.softmax(scores.float(), dim=-1).to(q.dtype)
+        att = self.dropout(att)
+        return att @ v
+    def _sage_attention(self, q, k, v):
+        if self.sageattn is None or not q.is_cuda:
+            if self.torch_fallback:
+                return None
+            raise RuntimeError("SageAttention requires CUDA + sageattention")
+        return self.sageattn(q.contiguous(), k.contiguous(), v.contiguous(), tensor_layout="HND", is_causal=True)
+    def _flash2_attention(self, q, k, v):
+        if self.flash_kernel is None or not q.is_cuda:
+            if self.torch_fallback:
+                return None
+            raise RuntimeError("FlashAttention2 requires CUDA + kernels")
+        q = q.transpose(1, 2).contiguous()
+        k = k.transpose(1, 2).contiguous()
+        v = v.transpose(1, 2).contiguous()
+        dropout_p = self.dropout_p if self.training else 0.0
+        y = self.flash_kernel.flash_attn_func(q, k, v, dropout_p=dropout_p, causal=True)
+        return y.transpose(1, 2).contiguous()
+    def _flash3_attention(self, q, k, v):
+        if self.flash_kernel is None or not q.is_cuda:
+            if self.torch_fallback:
+                return None
+            raise RuntimeError("FlashAttention3 requires CUDA + kernels")
+        q = q.transpose(1, 2).contiguous()
+        k = k.transpose(1, 2).contiguous()
+        v = v.transpose(1, 2).contiguous()
+        y = self.flash_kernel.flash_attn_func(q, k, v, causal=True)
+        return y.transpose(1, 2).contiguous()
+    def forward(self, x):
+        b, t, c = x.shape
+        qkv = self.qkv(x)
+        q, k, v = qkv.chunk(3, dim=-1)
+        q = q.view(b, t, self.n_head, self.head_dim).transpose(1, 2).contiguous()
+        k = k.view(b, t, self.n_head, self.head_dim).transpose(1, 2).contiguous()
+        v = v.view(b, t, self.n_head, self.head_dim).transpose(1, 2).contiguous()
+        if self.attention_backend == "sage":
+            y = self._sage_attention(q, k, v)
+            if y is None:
+                y = self._torch_attention(q, k, v, t)
+        elif self.attention_backend == "flash2":
+            y = self._flash2_attention(q, k, v)
+            if y is None:
+                y = self._torch_attention(q, k, v, t)
+        elif self.attention_backend == "flash3":
+            y = self._flash3_attention(q, k, v)
+            if y is None:
+                y = self._torch_attention(q, k, v, t)
+        else:
+            y = self._torch_attention(q, k, v, t)
+        y = y.transpose(1, 2).contiguous().view(b, t, c)
+        return self.proj(y)
+class MLP(nn.Module):
+    def __init__(self, config: TinyGPTConfig):
+        super().__init__()
+        self.fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
+        self.proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
+        self.dropout = nn.Dropout(config.dropout)
+    def forward(self, x):
+        x = self.fc(x)
+        x = F.gelu(x)
+        x = self.proj(x)
+        x = self.dropout(x)
+        return x
+class Block(nn.Module):
+    def __init__(self, config: TinyGPTConfig):
+        super().__init__()
+        self.ln1 = nn.LayerNorm(config.n_embd)
+        self.attn = CausalSelfAttention(config)
+        self.ln2 = nn.LayerNorm(config.n_embd)
+        self.mlp = MLP(config)
+    def forward(self, x):
+        x = x + self.attn(self.ln1(x))
+        x = x + self.mlp(self.ln2(x))
+        return x
+class TinyGPTPreTrainedModel(PreTrainedModel):
+    config_class = TinyGPTConfig
+    base_model_prefix = "tiny_gpt"
+    supports_gradient_checkpointing = False
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            nn.init.normal_(module.weight, mean=0.0, std=0.02)
+class TinyGPTModel(TinyGPTPreTrainedModel):
+    _tied_weights_keys = ["head.weight"]
+    def __init__(self, config: TinyGPTConfig):
+        super().__init__(config)
+        self.tok_emb = nn.Embedding(config.vocab_size, config.n_embd)
+        self.pos_emb = nn.Embedding(config.ctx_len, config.n_embd)
+        self.drop = nn.Dropout(config.dropout)
+        self.blocks = nn.ModuleList([Block(config) for _ in range(config.n_layer)])
+        self.ln_f = nn.LayerNorm(config.n_embd)
+        self.head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        self.post_init()
+        self.tie_weights()
+    def get_input_embeddings(self):
+        return self.tok_emb
+    def set_input_embeddings(self, value):
+        self.tok_emb = value
+        self.tie_weights()
+    def get_output_embeddings(self):
+        return self.head
+    def set_output_embeddings(self, new_embeddings):
+        self.head = new_embeddings
+    def tie_weights(self, *args, **kwargs):
+        self._tie_or_clone_weights(self.head, self.tok_emb)
+    def forward(self, input_ids, attention_mask=None, return_dict=True, return_logits=False, **kwargs):
+        b, t = input_ids.shape
+        if t > self.config.ctx_len:
+            raise ValueError(f"Input length {t} > ctx_len {self.config.ctx_len}. Truncate before calling the model.")
+        pos = torch.arange(0, t, dtype=torch.long, device=input_ids.device).unsqueeze(0)
+        x = self.tok_emb(input_ids) + self.pos_emb(pos)
+        x = self.drop(x)
+        for block in self.blocks:
+            x = block(x)
+        hidden = self.ln_f(x)
+        logits = self.head(hidden) if return_logits else None
+        if not return_dict:
+            return (hidden, logits) if return_logits else (hidden,)
+        if return_logits:
+            return hidden, logits
+        return BaseModelOutputWithPast(last_hidden_state=hidden, past_key_values=None, hidden_states=None, attentions=None)
+class TinyGPTForCausalLM(TinyGPTPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = ["tiny_gpt.head.weight"]
+    def __init__(self, config: TinyGPTConfig):
+        super().__init__(config)
+        self.tiny_gpt = TinyGPTModel(config)
+        self.post_init()
+        self.tie_weights()
+    def get_input_embeddings(self):
+        return self.tiny_gpt.tok_emb
+    def set_input_embeddings(self, value):
+        self.tiny_gpt.tok_emb = value
+        self.tie_weights()
+    def get_output_embeddings(self):
+        return self.tiny_gpt.head
+    def set_output_embeddings(self, new_embeddings):
+        self.tiny_gpt.head = new_embeddings
+    def tie_weights(self, *args, **kwargs):
+        self._tie_or_clone_weights(self.tiny_gpt.head, self.tiny_gpt.tok_emb)
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        return {"input_ids": input_ids}
+    def forward(self, input_ids, attention_mask=None, labels=None, return_dict=True, **kwargs):
+        hidden, logits = self.tiny_gpt(input_ids=input_ids, attention_mask=attention_mask, return_dict=True, return_logits=True)
+        loss = None
+        if labels is not None:
+            shift_logits = logits[:, :-1, :].contiguous()
+            shift_labels = labels[:, 1:].contiguous()
+            loss = F.cross_entropy(shift_logits.view(-1, shift_logits.size(-1)).float(), shift_labels.view(-1))
+        if not return_dict:
+            return ((loss, logits) if loss is not None else (logits,))
+        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=None, hidden_states=None, attentions=None)

requirements.txt

@@ -0,0 +1,4 @@
+torch
+transformers
+safetensors
+tokenizers

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "unk_token": "[UNK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]"
+}

tokenizer.json

@@ -0,0 +1,1467 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 0,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "[BOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": "[EOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFC"
+      },
+      {
+        "type": "Lowercase"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[BOS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[EOS]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "[BOS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[EOS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[EOS]",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[BOS]": {
+        "id": "[BOS]",
+        "ids": [
+          5
+        ],
+        "tokens": [
+          "[BOS]"
+        ]
+      },
+      "[EOS]": {
+        "id": "[EOS]",
+        "ids": [
+          6
+        ],
+        "tokens": [
+          "[EOS]"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[SEP]": 2,
+      "[CLS]": 3,
+      "[MASK]": 4,
+      "[BOS]": 5,
+      "[EOS]": 6,
+      "!": 7,
+      "\"": 8,
+      "#": 9,
+      "$": 10,
+      "%": 11,
+      "&": 12,
+      "'": 13,
+      "(": 14,
+      ")": 15,
+      "*": 16,
+      "+": 17,
+      ",": 18,
+      "-": 19,
+      ".": 20,
+      "/": 21,
+      "0": 22,
+      "1": 23,
+      "2": 24,
+      "3": 25,
+      "4": 26,
+      "5": 27,
+      "6": 28,
+      "7": 29,
+      "8": 30,
+      "9": 31,
+      ":": 32,
+      ";": 33,
+      "<": 34,
+      "=": 35,
+      ">": 36,
+      "?": 37,
+      "@": 38,
+      "A": 39,
+      "B": 40,
+      "C": 41,
+      "D": 42,
+      "E": 43,
+      "F": 44,
+      "G": 45,
+      "H": 46,
+      "I": 47,
+      "J": 48,
+      "K": 49,
+      "L": 50,
+      "M": 51,
+      "N": 52,
+      "O": 53,
+      "P": 54,
+      "Q": 55,
+      "R": 56,
+      "S": 57,
+      "T": 58,
+      "U": 59,
+      "V": 60,
+      "W": 61,
+      "X": 62,
+      "Y": 63,
+      "Z": 64,
+      "[": 65,
+      "\\": 66,
+      "]": 67,
+      "^": 68,
+      "_": 69,
+      "`": 70,
+      "a": 71,
+      "b": 72,
+      "c": 73,
+      "d": 74,
+      "e": 75,
+      "f": 76,
+      "g": 77,
+      "h": 78,
+      "i": 79,
+      "j": 80,
+      "k": 81,
+      "l": 82,
+      "m": 83,
+      "n": 84,
+      "o": 85,
+      "p": 86,
+      "q": 87,
+      "r": 88,
+      "s": 89,
+      "t": 90,
+      "u": 91,
+      "v": 92,
+      "w": 93,
+      "x": 94,
+      "y": 95,
+      "z": 96,
+      "{": 97,
+      "|": 98,
+      "}": 99,
+      "~": 100,
+      "¡": 101,
+      "¢": 102,
+      "£": 103,
+      "¤": 104,
+      "¥": 105,
+      "¦": 106,
+      "§": 107,
+      "¨": 108,
+      "©": 109,
+      "ª": 110,
+      "«": 111,
+      "¬": 112,
+      "®": 113,
+      "¯": 114,
+      "°": 115,
+      "±": 116,
+      "²": 117,
+      "³": 118,
+      "´": 119,
+      "µ": 120,
+      "¶": 121,
+      "·": 122,
+      "¸": 123,
+      "¹": 124,
+      "º": 125,
+      "»": 126,
+      "¼": 127,
+      "½": 128,
+      "¾": 129,
+      "¿": 130,
+      "À": 131,
+      "Á": 132,
+      "Â": 133,
+      "Ã": 134,
+      "Ä": 135,
+      "Å": 136,
+      "Æ": 137,
+      "Ç": 138,
+      "È": 139,
+      "É": 140,
+      "Ê": 141,
+      "Ë": 142,
+      "Ì": 143,
+      "Í": 144,
+      "Î": 145,
+      "Ï": 146,
+      "Ð": 147,
+      "Ñ": 148,
+      "Ò": 149,
+      "Ó": 150,
+      "Ô": 151,
+      "Õ": 152,
+      "Ö": 153,
+      "×": 154,
+      "Ø": 155,
+      "Ù": 156,
+      "Ú": 157,
+      "Û": 158,
+      "Ü": 159,
+      "Ý": 160,
+      "Þ": 161,
+      "ß": 162,
+      "à": 163,
+      "á": 164,
+      "â": 165,
+      "ã": 166,
+      "ä": 167,
+      "å": 168,
+      "æ": 169,
+      "ç": 170,
+      "è": 171,
+      "é": 172,
+      "ê": 173,
+      "ë": 174,
+      "ì": 175,
+      "í": 176,
+      "î": 177,
+      "ï": 178,
+      "ð": 179,
+      "ñ": 180,
+      "ò": 181,
+      "ó": 182,
+      "ô": 183,
+      "õ": 184,
+      "ö": 185,
+      "÷": 186,
+      "ø": 187,
+      "ù": 188,
+      "ú": 189,
+      "û": 190,
+      "ü": 191,
+      "ý": 192,
+      "þ": 193,
+      "ÿ": 194,
+      "Ā": 195,
+      "ā": 196,
+      "Ă": 197,
+      "ă": 198,
+      "Ą": 199,
+      "ą": 200,
+      "Ć": 201,
+      "ć": 202,
+      "Ĉ": 203,
+      "ĉ": 204,
+      "Ċ": 205,
+      "ċ": 206,
+      "Č": 207,
+      "č": 208,
+      "Ď": 209,
+      "ď": 210,
+      "Đ": 211,
+      "đ": 212,
+      "Ē": 213,
+      "ē": 214,
+      "Ĕ": 215,
+      "ĕ": 216,
+      "Ė": 217,
+      "ė": 218,
+      "Ę": 219,
+      "ę": 220,
+      "Ě": 221,
+      "ě": 222,
+      "Ĝ": 223,
+      "ĝ": 224,
+      "Ğ": 225,
+      "ğ": 226,
+      "Ġ": 227,
+      "ġ": 228,
+      "Ģ": 229,
+      "ģ": 230,
+      "Ĥ": 231,
+      "ĥ": 232,
+      "Ħ": 233,
+      "ħ": 234,
+      "Ĩ": 235,
+      "ĩ": 236,
+      "Ī": 237,
+      "ī": 238,
+      "Ĭ": 239,
+      "ĭ": 240,
+      "Į": 241,
+      "į": 242,
+      "İ": 243,
+      "ı": 244,
+      "IJ": 245,
+      "ij": 246,
+      "Ĵ": 247,
+      "ĵ": 248,
+      "Ķ": 249,
+      "ķ": 250,
+      "ĸ": 251,
+      "Ĺ": 252,
+      "ĺ": 253,
+      "Ļ": 254,
+      "ļ": 255,
+      "Ľ": 256,
+      "ľ": 257,
+      "Ŀ": 258,
+      "ŀ": 259,
+      "Ł": 260,
+      "ł": 261,
+      "Ń": 262,
+      "es": 263,
+      "en": 264,
+      "th": 265,
+      "the": 266,
+      "at": 267,
+      "or": 268,
+      "de": 269,
+      "ates": 270,
+      "ut": 271,
+      "co": 272,
+      "ra": 273,
+      "di": 274,
+      "Ġp": 275,
+      "Ġt": 276,
+      "ns": 277,
+      "ent": 278,
+      "st": 279,
+      "Ġpr": 280,
+      "Ġa": 281,
+      "io": 282,
+      "ts": 283,
+      "Ġde": 284,
+      "Ġdeco": 285,
+      "li": 286,
+      "wor": 287,
+      "work": 288,
+      "Ġf": 289,
+      "ne": 290,
+      "Ġpro": 291,
+      "Ġg": 292,
+      "des": 293,
+      "Ġe": 294,
+      "er": 295,
+      "put": 296,
+      "al": 297,
+      "der": 298,
+      "ces": 299,
+      "for": 300,
+      "rans": 301,
+      "Ġtrans": 302,
+      "form": 303,
+      "Ġtransform": 304,
+      "Ġs": 305,
+      "Ġo": 306,
+      "zes": 307,
+      "ions": 308,
+      "Ġen": 309,
+      "Ġenco": 310,
+      "re": 311,
+      "Ġdecodes": 312,
+      "ab": 313,
+      "but": 314,
+      "bab": 315,
+      "ili": 316,
+      "ibut": 317,
+      "ribut": 318,
+      "ty": 319,
+      "Ġdi": 320,
+      "stribut": 321,
+      "Ġprobab": 322,
+      "ility": 323,
+      "Ġdistribut": 324,
+      "Ġprobability": 325,
+      "Ġdistributions": 326,
+      "dd": 327,
+      "hi": 328,
+      "Ġst": 329,
+      "Ġhi": 330,
+      "dden": 331,
+      "Ġstates": 332,
+      "Ġhidden": 333,
+      "ener": 334,
+      "Ġgener": 335,
+      "Ġgenerates": 336,
+      "ework": 337,
+      "mework": 338,
+      "ramework": 339,
+      "Ġframework": 340,
+      "as": 341,
+      "cl": 342,
+      "fi": 343,
+      "ifi": 344,
+      "sifi": 345,
+      "Ġcl": 346,
+      "assifi": 347,
+      "Ġclassifi": 348,
+      "Ġclassifies": 349,
+      "be": 350,
+      "ddi": 351,
+      "gs": 352,
+      "mbe": 353,
+      "ngs": 354,
+      "Ġembe": 355,
+      "ddings": 356,
+      "Ġembeddings": 357,
+      "twork": 358,
+      "Ġne": 359,
+      "Ġnetwork": 360,
+      "eli": 361,
+      "ip": 362,
+      "Ġpip": 363,
+      "eline": 364,
+      "Ġpipeline": 365,
+      "Ġdecoder": 366,
+      "ct": 367,
+      "eat": 368,
+      "ect": 369,
+      "ure": 370,
+      "vect": 371,
+      "Ġvect": 372,
+      "ors": 373,
+      "Ġfeat": 374,
+      "Ġvectors": 375,
+      "Ġfeature": 376,
+      "an": 377,
+      "gent": 378,
+      "Ġagent": 379,
+      "mo": 380,
+      "Ġmo": 381,
+      "del": 382,
+      "Ġmodel": 383,
+      "in": 384,
+      "ken": 385,
+      "oken": 386,
+      "Ġin": 387,
+      "Ġtoken": 388,
+      "Ġinput": 389,
+      "Ġtokens": 390,
+      "gi": 391,
+      "lo": 392,
+      "Ġlo": 393,
+      "utput": 394,
+      "Ġoutput": 395,
+      "gits": 396,
+      "Ġlogits": 397,
+      "uates": 398,
+      "val": 399,
+      "Ġeval": 400,
+      "Ġtransforms": 401,
+      "Ġevaluates": 402,
+      "ses": 403,
+      "Ġproces": 404,
+      "Ġprocesses": 405,
+      "dates": 406,
+      "pdates": 407,
+      "updates": 408,
+      "Ġupdates": 409,
+      "radi": 410,
+      "Ġgradi": 411,
+      "Ġgradient": 412,
+      "nal": 413,
+      "yzes": 414,
+      "Ġanal": 415,
+      "Ġanalyzes": 416,
+      "eq": 417,
+      "ex": 418,
+      "uen": 419,
+      "Ġtex": 420,
+      "Ġseq": 421,
+      "uences": 422,
+      "Ġtext": 423,
+      "Ġsequences": 424,
+      "em": 425,
+      "yst": 426,
+      "Ġsyst": 427,
+      "Ġsystem": 428,
+      "Ġtransformer": 429,
+      "Ġencodes": 430,
+      "im": 431,
+      "izes": 432,
+      "pt": 433,
+      "Ġopt": 434,
+      "imizes": 435,
+      "Ġoptimizes": 436,
+      "Ġencoder": 437,
+      "ei": 438,
+      "gh": 439,
+      "tent": 440,
+      "wei": 441,
+      "Ġat": 442,
+      "Ġwei": 443,
+      "ion": 444,
+      "ghts": 445,
+      "tention": 446,
+      "Ġattention": 447,
+      "Ġweights": 448,
+      "gor": 449,
+      "ith": 450,
+      "lgor": 451,
+      "Ġalgor": 452,
+      "ithm": 453,
+      "Ġalgorithm": 454,
+      "cts": 455,
+      "edi": 456,
+      "Ġpredi": 457,
+      "Ġpredicts": 458,
+      "pr": 459,
+      "Ġre": 460,
+      "esent": 461,
+      "ations": 462,
+      "present": 463,
+      "Ġrepresent": 464,
+      "Ġrepresentations": 465
+    },
+    "merges": [
+      [
+        "e",
+        "s"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "t",
+        "h"
+      ],
+      [
+        "th",
+        "e"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "d",
+        "e"
+      ],
+      [
+        "at",
+        "es"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "c",
+        "o"
+      ],
+      [
+        "r",
+        "a"
+      ],
+      [
+        "d",
+        "i"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "n",
+        "s"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "Ġp",
+        "r"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "i",
+        "o"
+      ],
+      [
+        "t",
+        "s"
+      ],
+      [
+        "Ġ",
+        "de"
+      ],
+      [
+        "Ġde",
+        "co"
+      ],
+      [
+        "l",
+        "i"
+      ],
+      [
+        "w",
+        "or"
+      ],
+      [
+        "wor",
+        "k"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "n",
+        "e"
+      ],
+      [
+        "Ġpr",
+        "o"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "d",
+        "es"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "p",
+        "ut"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "de",
+        "r"
+      ],
+      [
+        "c",
+        "es"
+      ],
+      [
+        "f",
+        "or"
+      ],
+      [
+        "ra",
+        "ns"
+      ],
+      [
+        "Ġt",
+        "rans"
+      ],
+      [
+        "for",
+        "m"
+      ],
+      [
+        "Ġtrans",
+        "form"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "z",
+        "es"
+      ],
+      [
+        "io",
+        "ns"
+      ],
+      [
+        "Ġ",
+        "en"
+      ],
+      [
+        "Ġen",
+        "co"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "Ġdeco",
+        "des"
+      ],
+      [
+        "a",
+        "b"
+      ],
+      [
+        "b",
+        "ut"
+      ],
+      [
+        "b",
+        "ab"
+      ],
+      [
+        "i",
+        "li"
+      ],
+      [
+        "i",
+        "but"
+      ],
+      [
+        "r",
+        "ibut"
+      ],
+      [
+        "t",
+        "y"
+      ],
+      [
+        "Ġ",
+        "di"
+      ],
+      [
+        "st",
+        "ribut"
+      ],
+      [
+        "Ġpro",
+        "bab"
+      ],
+      [
+        "ili",
+        "ty"
+      ],
+      [
+        "Ġdi",
+        "stribut"
+      ],
+      [
+        "Ġprobab",
+        "ility"
+      ],
+      [
+        "Ġdistribut",
+        "ions"
+      ],
+      [
+        "d",
+        "d"
+      ],
+      [
+        "h",
+        "i"
+      ],
+      [
+        "Ġ",
+        "st"
+      ],
+      [
+        "Ġ",
+        "hi"
+      ],
+      [
+        "dd",
+        "en"
+      ],
+      [
+        "Ġst",
+        "ates"
+      ],
+      [
+        "Ġhi",
+        "dden"
+      ],
+      [
+        "en",
+        "er"
+      ],
+      [
+        "Ġg",
+        "ener"
+      ],
+      [
+        "Ġgener",
+        "ates"
+      ],
+      [
+        "e",
+        "work"
+      ],
+      [
+        "m",
+        "ework"
+      ],
+      [
+        "ra",
+        "mework"
+      ],
+      [
+        "Ġf",
+        "ramework"
+      ],
+      [
+        "a",
+        "s"
+      ],
+      [
+        "c",
+        "l"
+      ],
+      [
+        "f",
+        "i"
+      ],
+      [
+        "i",
+        "fi"
+      ],
+      [
+        "s",
+        "ifi"
+      ],
+      [
+        "Ġ",
+        "cl"
+      ],
+      [
+        "as",
+        "sifi"
+      ],
+      [
+        "Ġcl",
+        "assifi"
+      ],
+      [
+        "Ġclassifi",
+        "es"
+      ],
+      [
+        "b",
+        "e"
+      ],
+      [
+        "d",
+        "di"
+      ],
+      [
+        "g",
+        "s"
+      ],
+      [
+        "m",
+        "be"
+      ],
+      [
+        "n",
+        "gs"
+      ],
+      [
+        "Ġe",
+        "mbe"
+      ],
+      [
+        "ddi",
+        "ngs"
+      ],
+      [
+        "Ġembe",
+        "ddings"
+      ],
+      [
+        "t",
+        "work"
+      ],
+      [
+        "Ġ",
+        "ne"
+      ],
+      [
+        "Ġne",
+        "twork"
+      ],
+      [
+        "e",
+        "li"
+      ],
+      [
+        "i",
+        "p"
+      ],
+      [
+        "Ġp",
+        "ip"
+      ],
+      [
+        "eli",
+        "ne"
+      ],
+      [
+        "Ġpip",
+        "eline"
+      ],
+      [
+        "Ġdeco",
+        "der"
+      ],
+      [
+        "c",
+        "t"
+      ],
+      [
+        "e",
+        "at"
+      ],
+      [
+        "e",
+        "ct"
+      ],
+      [
+        "u",
+        "re"
+      ],
+      [
+        "v",
+        "ect"
+      ],
+      [
+        "Ġ",
+        "vect"
+      ],
+      [
+        "or",
+        "s"
+      ],
+      [
+        "Ġf",
+        "eat"
+      ],
+      [
+        "Ġvect",
+        "ors"
+      ],
+      [
+        "Ġfeat",
+        "ure"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "g",
+        "ent"
+      ],
+      [
+        "Ġa",
+        "gent"
+      ],
+      [
+        "m",
+        "o"
+      ],
+      [
+        "Ġ",
+        "mo"
+      ],
+      [
+        "de",
+        "l"
+      ],
+      [
+        "Ġmo",
+        "del"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "k",
+        "en"
+      ],
+      [
+        "o",
+        "ken"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "Ġt",
+        "oken"
+      ],
+      [
+        "Ġin",
+        "put"
+      ],
+      [
+        "Ġtoken",
+        "s"
+      ],
+      [
+        "g",
+        "i"
+      ],
+      [
+        "l",
+        "o"
+      ],
+      [
+        "Ġ",
+        "lo"
+      ],
+      [
+        "ut",
+        "put"
+      ],
+      [
+        "Ġo",
+        "utput"
+      ],
+      [
+        "gi",
+        "ts"
+      ],
+      [
+        "Ġlo",
+        "gits"
+      ],
+      [
+        "u",
+        "ates"
+      ],
+      [
+        "v",
+        "al"
+      ],
+      [
+        "Ġe",
+        "val"
+      ],
+      [
+        "Ġtransform",
+        "s"
+      ],
+      [
+        "Ġeval",
+        "uates"
+      ],
+      [
+        "s",
+        "es"
+      ],
+      [
+        "Ġpro",
+        "ces"
+      ],
+      [
+        "Ġproces",
+        "ses"
+      ],
+      [
+        "d",
+        "ates"
+      ],
+      [
+        "p",
+        "dates"
+      ],
+      [
+        "u",
+        "pdates"
+      ],
+      [
+        "Ġ",
+        "updates"
+      ],
+      [
+        "ra",
+        "di"
+      ],
+      [
+        "Ġg",
+        "radi"
+      ],
+      [
+        "Ġgradi",
+        "ent"
+      ],
+      [
+        "n",
+        "al"
+      ],
+      [
+        "y",
+        "zes"
+      ],
+      [
+        "Ġa",
+        "nal"
+      ],
+      [
+        "Ġanal",
+        "yzes"
+      ],
+      [
+        "e",
+        "q"
+      ],
+      [
+        "e",
+        "x"
+      ],
+      [
+        "u",
+        "en"
+      ],
+      [
+        "Ġt",
+        "ex"
+      ],
+      [
+        "Ġs",
+        "eq"
+      ],
+      [
+        "uen",
+        "ces"
+      ],
+      [
+        "Ġtex",
+        "t"
+      ],
+      [
+        "Ġseq",
+        "uences"
+      ],
+      [
+        "e",
+        "m"
+      ],
+      [
+        "y",
+        "st"
+      ],
+      [
+        "Ġs",
+        "yst"
+      ],
+      [
+        "Ġsyst",
+        "em"
+      ],
+      [
+        "Ġtransform",
+        "er"
+      ],
+      [
+        "Ġenco",
+        "des"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "i",
+        "zes"
+      ],
+      [
+        "p",
+        "t"
+      ],
+      [
+        "Ġo",
+        "pt"
+      ],
+      [
+        "im",
+        "izes"
+      ],
+      [
+        "Ġopt",
+        "imizes"
+      ],
+      [
+        "Ġenco",
+        "der"
+      ],
+      [
+        "e",
+        "i"
+      ],
+      [
+        "g",
+        "h"
+      ],
+      [
+        "t",
+        "ent"
+      ],
+      [
+        "w",
+        "ei"
+      ],
+      [
+        "Ġ",
+        "at"
+      ],
+      [
+        "Ġ",
+        "wei"
+      ],
+      [
+        "io",
+        "n"
+      ],
+      [
+        "gh",
+        "ts"
+      ],
+      [
+        "tent",
+        "ion"
+      ],
+      [
+        "Ġat",
+        "tention"
+      ],
+      [
+        "Ġwei",
+        "ghts"
+      ],
+      [
+        "g",
+        "or"
+      ],
+      [
+        "i",
+        "th"
+      ],
+      [
+        "l",
+        "gor"
+      ],
+      [
+        "Ġa",
+        "lgor"
+      ],
+      [
+        "ith",
+        "m"
+      ],
+      [
+        "Ġalgor",
+        "ithm"
+      ],
+      [
+        "c",
+        "ts"
+      ],
+      [
+        "e",
+        "di"
+      ],
+      [
+        "Ġpr",
+        "edi"
+      ],
+      [
+        "Ġpredi",
+        "cts"
+      ],
+      [
+        "p",
+        "r"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "es",
+        "ent"
+      ],
+      [
+        "at",
+        "ions"
+      ],
+      [
+        "pr",
+        "esent"
+      ],
+      [
+        "Ġre",
+        "present"
+      ],
+      [
+        "Ġrepresent",
+        "ations"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "clean_up_tokenization_spaces": false,
+  "padding_side": "right",
+  "truncation_side": "right",
+  "unk_token": "[UNK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "model_max_length": 128
+}

vocab.json

@@ -0,0 +1,468 @@
+{
+  "[PAD]": 0,
+  "[UNK]": 1,
+  "[SEP]": 2,
+  "[CLS]": 3,
+  "[MASK]": 4,
+  "[BOS]": 5,
+  "[EOS]": 6,
+  "!": 7,
+  "\"": 8,
+  "#": 9,
+  "$": 10,
+  "%": 11,
+  "&": 12,
+  "'": 13,
+  "(": 14,
+  ")": 15,
+  "*": 16,
+  "+": 17,
+  ",": 18,
+  "-": 19,
+  ".": 20,
+  "/": 21,
+  "0": 22,
+  "1": 23,
+  "2": 24,
+  "3": 25,
+  "4": 26,
+  "5": 27,
+  "6": 28,
+  "7": 29,
+  "8": 30,
+  "9": 31,
+  ":": 32,
+  ";": 33,
+  "<": 34,
+  "=": 35,
+  ">": 36,
+  "?": 37,
+  "@": 38,
+  "A": 39,
+  "B": 40,
+  "C": 41,
+  "D": 42,
+  "E": 43,
+  "F": 44,
+  "G": 45,
+  "H": 46,
+  "I": 47,
+  "J": 48,
+  "K": 49,
+  "L": 50,
+  "M": 51,
+  "N": 52,
+  "O": 53,
+  "P": 54,
+  "Q": 55,
+  "R": 56,
+  "S": 57,
+  "T": 58,
+  "U": 59,
+  "V": 60,
+  "W": 61,
+  "X": 62,
+  "Y": 63,
+  "Z": 64,
+  "[": 65,
+  "\\": 66,
+  "]": 67,
+  "^": 68,
+  "_": 69,
+  "`": 70,
+  "a": 71,
+  "b": 72,
+  "c": 73,
+  "d": 74,
+  "e": 75,
+  "f": 76,
+  "g": 77,
+  "h": 78,
+  "i": 79,
+  "j": 80,
+  "k": 81,
+  "l": 82,
+  "m": 83,
+  "n": 84,
+  "o": 85,
+  "p": 86,
+  "q": 87,
+  "r": 88,
+  "s": 89,
+  "t": 90,
+  "u": 91,
+  "v": 92,
+  "w": 93,
+  "x": 94,
+  "y": 95,
+  "z": 96,
+  "{": 97,
+  "|": 98,
+  "}": 99,
+  "~": 100,
+  "¡": 101,
+  "¢": 102,
+  "£": 103,
+  "¤": 104,
+  "¥": 105,
+  "¦": 106,
+  "§": 107,
+  "¨": 108,
+  "©": 109,
+  "ª": 110,
+  "«": 111,
+  "¬": 112,
+  "®": 113,
+  "¯": 114,
+  "°": 115,
+  "±": 116,
+  "²": 117,
+  "³": 118,
+  "´": 119,
+  "µ": 120,
+  "¶": 121,
+  "·": 122,
+  "¸": 123,
+  "¹": 124,
+  "º": 125,
+  "»": 126,
+  "¼": 127,
+  "½": 128,
+  "¾": 129,
+  "¿": 130,
+  "À": 131,
+  "Á": 132,
+  "Â": 133,
+  "Ã": 134,
+  "Ä": 135,
+  "Å": 136,
+  "Æ": 137,
+  "Ç": 138,
+  "È": 139,
+  "É": 140,
+  "Ê": 141,
+  "Ë": 142,
+  "Ì": 143,
+  "Í": 144,
+  "Î": 145,
+  "Ï": 146,
+  "Ð": 147,
+  "Ñ": 148,
+  "Ò": 149,
+  "Ó": 150,
+  "Ô": 151,
+  "Õ": 152,
+  "Ö": 153,
+  "×": 154,
+  "Ø": 155,
+  "Ù": 156,
+  "Ú": 157,
+  "Û": 158,
+  "Ü": 159,
+  "Ý": 160,
+  "Þ": 161,
+  "ß": 162,
+  "à": 163,
+  "á": 164,
+  "â": 165,
+  "ã": 166,
+  "ä": 167,
+  "å": 168,
+  "æ": 169,
+  "ç": 170,
+  "è": 171,
+  "é": 172,
+  "ê": 173,
+  "ë": 174,
+  "ì": 175,
+  "í": 176,
+  "î": 177,
+  "ï": 178,
+  "ð": 179,
+  "ñ": 180,
+  "ò": 181,
+  "ó": 182,
+  "ô": 183,
+  "õ": 184,
+  "ö": 185,
+  "÷": 186,
+  "ø": 187,
+  "ù": 188,
+  "ú": 189,
+  "û": 190,
+  "ü": 191,
+  "ý": 192,
+  "þ": 193,
+  "ÿ": 194,
+  "Ā": 195,
+  "ā": 196,
+  "Ă": 197,
+  "ă": 198,
+  "Ą": 199,
+  "ą": 200,
+  "Ć": 201,
+  "ć": 202,
+  "Ĉ": 203,
+  "ĉ": 204,
+  "Ċ": 205,
+  "ċ": 206,
+  "Č": 207,
+  "č": 208,
+  "Ď": 209,
+  "ď": 210,
+  "Đ": 211,
+  "đ": 212,
+  "Ē": 213,
+  "ē": 214,
+  "Ĕ": 215,
+  "ĕ": 216,
+  "Ė": 217,
+  "ė": 218,
+  "Ę": 219,
+  "ę": 220,
+  "Ě": 221,
+  "ě": 222,
+  "Ĝ": 223,
+  "ĝ": 224,
+  "Ğ": 225,
+  "ğ": 226,
+  "Ġ": 227,
+  "ġ": 228,
+  "Ģ": 229,
+  "ģ": 230,
+  "Ĥ": 231,
+  "ĥ": 232,
+  "Ħ": 233,
+  "ħ": 234,
+  "Ĩ": 235,
+  "ĩ": 236,
+  "Ī": 237,
+  "ī": 238,
+  "Ĭ": 239,
+  "ĭ": 240,
+  "Į": 241,
+  "į": 242,
+  "İ": 243,
+  "ı": 244,
+  "IJ": 245,
+  "ij": 246,
+  "Ĵ": 247,
+  "ĵ": 248,
+  "Ķ": 249,
+  "ķ": 250,
+  "ĸ": 251,
+  "Ĺ": 252,
+  "ĺ": 253,
+  "Ļ": 254,
+  "ļ": 255,
+  "Ľ": 256,
+  "ľ": 257,
+  "Ŀ": 258,
+  "ŀ": 259,
+  "Ł": 260,
+  "ł": 261,
+  "Ń": 262,
+  "es": 263,
+  "en": 264,
+  "th": 265,
+  "the": 266,
+  "at": 267,
+  "or": 268,
+  "de": 269,
+  "ates": 270,
+  "ut": 271,
+  "co": 272,
+  "ra": 273,
+  "di": 274,
+  "Ġp": 275,
+  "Ġt": 276,
+  "ns": 277,
+  "ent": 278,
+  "st": 279,
+  "Ġpr": 280,
+  "Ġa": 281,
+  "io": 282,
+  "ts": 283,
+  "Ġde": 284,
+  "Ġdeco": 285,
+  "li": 286,
+  "wor": 287,
+  "work": 288,
+  "Ġf": 289,
+  "ne": 290,
+  "Ġpro": 291,
+  "Ġg": 292,
+  "des": 293,
+  "Ġe": 294,
+  "er": 295,
+  "put": 296,
+  "al": 297,
+  "der": 298,
+  "ces": 299,
+  "for": 300,
+  "rans": 301,
+  "Ġtrans": 302,
+  "form": 303,
+  "Ġtransform": 304,
+  "Ġs": 305,
+  "Ġo": 306,
+  "zes": 307,
+  "ions": 308,
+  "Ġen": 309,
+  "Ġenco": 310,
+  "re": 311,
+  "Ġdecodes": 312,
+  "ab": 313,
+  "but": 314,
+  "bab": 315,
+  "ili": 316,
+  "ibut": 317,
+  "ribut": 318,
+  "ty": 319,
+  "Ġdi": 320,
+  "stribut": 321,
+  "Ġprobab": 322,
+  "ility": 323,
+  "Ġdistribut": 324,
+  "Ġprobability": 325,
+  "Ġdistributions": 326,
+  "dd": 327,
+  "hi": 328,
+  "Ġst": 329,
+  "Ġhi": 330,
+  "dden": 331,
+  "Ġstates": 332,
+  "Ġhidden": 333,
+  "ener": 334,
+  "Ġgener": 335,
+  "Ġgenerates": 336,
+  "ework": 337,
+  "mework": 338,
+  "ramework": 339,
+  "Ġframework": 340,
+  "as": 341,
+  "cl": 342,
+  "fi": 343,
+  "ifi": 344,
+  "sifi": 345,
+  "Ġcl": 346,
+  "assifi": 347,
+  "Ġclassifi": 348,
+  "Ġclassifies": 349,
+  "be": 350,
+  "ddi": 351,
+  "gs": 352,
+  "mbe": 353,
+  "ngs": 354,
+  "Ġembe": 355,
+  "ddings": 356,
+  "Ġembeddings": 357,
+  "twork": 358,
+  "Ġne": 359,
+  "Ġnetwork": 360,
+  "eli": 361,
+  "ip": 362,
+  "Ġpip": 363,
+  "eline": 364,
+  "Ġpipeline": 365,
+  "Ġdecoder": 366,
+  "ct": 367,
+  "eat": 368,
+  "ect": 369,
+  "ure": 370,
+  "vect": 371,
+  "Ġvect": 372,
+  "ors": 373,
+  "Ġfeat": 374,
+  "Ġvectors": 375,
+  "Ġfeature": 376,
+  "an": 377,
+  "gent": 378,
+  "Ġagent": 379,
+  "mo": 380,
+  "Ġmo": 381,
+  "del": 382,
+  "Ġmodel": 383,
+  "in": 384,
+  "ken": 385,
+  "oken": 386,
+  "Ġin": 387,
+  "Ġtoken": 388,
+  "Ġinput": 389,
+  "Ġtokens": 390,
+  "gi": 391,
+  "lo": 392,
+  "Ġlo": 393,
+  "utput": 394,
+  "Ġoutput": 395,
+  "gits": 396,
+  "Ġlogits": 397,
+  "uates": 398,
+  "val": 399,
+  "Ġeval": 400,
+  "Ġtransforms": 401,
+  "Ġevaluates": 402,
+  "ses": 403,
+  "Ġproces": 404,
+  "Ġprocesses": 405,
+  "dates": 406,
+  "pdates": 407,
+  "updates": 408,
+  "Ġupdates": 409,
+  "radi": 410,
+  "Ġgradi": 411,
+  "Ġgradient": 412,
+  "nal": 413,
+  "yzes": 414,
+  "Ġanal": 415,
+  "Ġanalyzes": 416,
+  "eq": 417,
+  "ex": 418,
+  "uen": 419,
+  "Ġtex": 420,
+  "Ġseq": 421,
+  "uences": 422,
+  "Ġtext": 423,
+  "Ġsequences": 424,
+  "em": 425,
+  "yst": 426,
+  "Ġsyst": 427,
+  "Ġsystem": 428,
+  "Ġtransformer": 429,
+  "Ġencodes": 430,
+  "im": 431,
+  "izes": 432,
+  "pt": 433,
+  "Ġopt": 434,
+  "imizes": 435,
+  "Ġoptimizes": 436,
+  "Ġencoder": 437,
+  "ei": 438,
+  "gh": 439,
+  "tent": 440,
+  "wei": 441,
+  "Ġat": 442,
+  "Ġwei": 443,
+  "ion": 444,
+  "ghts": 445,
+  "tention": 446,
+  "Ġattention": 447,
+  "Ġweights": 448,
+  "gor": 449,
+  "ith": 450,
+  "lgor": 451,
+  "Ġalgor": 452,
+  "ithm": 453,
+  "Ġalgorithm": 454,
+  "cts": 455,
+  "edi": 456,
+  "Ġpredi": 457,
+  "Ġpredicts": 458,
+  "pr": 459,
+  "Ġre": 460,
+  "esent": 461,
+  "ations": 462,
+  "present": 463,
+  "Ġrepresent": 464,
+  "Ġrepresentations": 465
+}