Upload 9 files

config.json

@@ -0,0 +1,48 @@
+{
+    "architectures": [
+        "DeweyV1"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "auto_map": {
+        "AutoModel": "modeling_dewey_v1.DeweyV1"
+    },
+    "bos_token_id": 50281,
+    "classifier_activation": "gelu",
+    "classifier_bias": false,
+    "classifier_dropout": 0.0,
+    "classifier_pooling": "mean",
+    "cls_token_id": 50281,
+    "decoder_bias": true,
+    "deterministic_flash_attn": false,
+    "embedding_dropout": 0.0,
+    "eos_token_id": 50282,
+    "global_attn_every_n_layers": 3,
+    "global_rope_theta": 73780400,
+    "gradient_checkpointing": false,
+    "hidden_activation": "gelu",
+    "hidden_size": 1024,
+    "initializer_cutoff_factor": 2.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 2624,
+    "layer_norm_eps": 1e-05,
+    "local_attention": 128,
+    "local_rope_theta": 10000.0,
+    "max_position_embeddings": 131072,
+    "mlp_bias": false,
+    "mlp_dropout": 0.0,
+    "model_type": "modernbert",
+    "norm_bias": false,
+    "norm_eps": 1e-05,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "pad_token_id": 50283,
+    "position_embedding_type": "absolute",
+    "sep_token_id": 50282,
+    "torch_dtype": "float32",
+    "transformers_version": "4.49.0",
+    "vector_size": 2048,
+    "single_vector_type":"cls_add_mean",
+    "vocab_size": 50370,
+    "tie_word_embeddings":false
+}

custom_st.py

@@ -0,0 +1,67 @@
+import torch
+from typing import Optional
+from pydantic import BaseModel
+from sentence_transformers.models import Transformer as BaseTransformer
+
+
+class TextSpan(BaseModel):
+    s: int
+    e: int
+    module_name: str
+    text: Optional[str] = None
+
+
+class DeweyTransformer(BaseTransformer):
+    def __init__(
+            self,
+            model_name_or_path: str,
+            **kwargs,
+    ):
+        self.single_vector_type = kwargs.get("config_args", {}).get("single_vector_type", "mean")
+        super().__init__(model_name_or_path, **kwargs)
+
+    def forward(
+            self, features: dict[str, torch.Tensor], **kwargs
+    ) -> dict[str, torch.Tensor]:
+        prompt_length = features.get("prompt_length", 0)
+        if prompt_length > 0:
+            # in MondernBert, text is surrounded by [CLS] and [SEP]
+            prompt_length -= 1
+        batch_text_spans = []
+        for data_len in features["attention_mask"].sum(dim=1):
+            if self.single_vector_type == "cls":
+                batch_text_spans.append(
+                    [
+                        TextSpan(s=0, e=1, module_name="cls_linear")
+                    ]
+                )
+            elif self.single_vector_type == "mean":
+                batch_text_spans.append(
+                    [
+                        TextSpan(s=1 + prompt_length, e=data_len - 1, module_name="chunk_linear")
+                    ]
+                )
+            elif self.single_vector_type == "cls_add_mean":
+                batch_text_spans.append(
+                    [
+                        TextSpan(s=0, e=1, module_name="cls_linear"),
+                        TextSpan(s=1 + prompt_length, e=data_len - 1, module_name="chunk_linear")
+                    ]
+                )
+            else:
+                raise Exception("single_vector_type should be in {cls, mean or cls_add_mean}")
+
+        trans_features = {
+            "input_ids": features["input_ids"],
+            "attention_mask": features["attention_mask"],
+            "batch_text_spans": batch_text_spans,
+            "normalize_embeddings": self.single_vector_type == "cls_add_mean",
+        }
+        # print(features["input_ids"].shape)
+        vectors_list = self.auto_model(**trans_features, **kwargs)
+        sentence_embedding = torch.cat(
+            [vecs.mean(dim=0, keepdim=True) for vecs in vectors_list],
+            dim=0
+        )
+        features.update({"sentence_embedding": sentence_embedding})
+        return features

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b474c19520443c3a144d1c62e4d889d1de2580b67b3c9aebb31e24c5c2acac8
+size 1595946872

modeling_dewey_v1.py

@@ -0,0 +1,283 @@
+import logging
+import torch
+import numpy as np
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Union, Optional, Tuple, List
+from pydantic import BaseModel
+from tqdm import tqdm
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from transformers import ModernBertModel, ModernBertPreTrainedModel, ModernBertConfig
+
+
+class TextSpan(BaseModel):
+    s: int
+    e: int
+    module_name: str
+    text: Optional[str] = None
+
+
+class Instance(BaseModel):
+    original_text: str
+    text_spans: List[TextSpan]
+
+
+def recursive_split(text, chunk_size=256, chunk_overlap=32):
+    """ recursive split a text by RecursiveCharacterTextSplitter in langchain_text_splitters """
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=lambda x: len(x.split()),
+        separators=["\n\n", "\n", ". ", "? ", "! ", "; "],
+    )
+    chunks = splitter.split_text(text)
+    if not chunks:
+        logging.error(f"Error, chunks is empty, text:{text}")
+        return [text], [[0, len(text)]]
+    chunk_span = [
+        # TODO a text may have multi same chunks
+        [text.find(chunk), text.find(chunk) + len(chunk)]
+        for chunk in chunks
+    ]
+    assert chunk_span[0][0] == 0
+    assert all((span[0] >= 0 for span in chunk_span))
+    return chunks, chunk_span
+
+
+def make_batch_input_for_prediction(
+        texts: List[str],
+        tokenizer,
+        max_seq_length: int,
+        chunk_size=256,
+        chunk_overlap=32,
+        prompt: str = "",
+        fast_chunk: bool = False,
+        batch_text_spans: List[List[TextSpan]] = None,
+):
+    """ prepare input"""
+    if batch_text_spans is not None:
+        ipt = tokenizer(
+            [prompt + i for i in texts],
+            padding="longest",
+            truncation=True,
+            max_length=max_seq_length,
+            return_tensors="pt"
+        )
+        for text_spans, data_len in zip(batch_text_spans, ipt["attention_mask"].sum(dim=1)):
+            for text_span in text_spans:
+                assert -1 < text_span.s < text_span.e <= data_len
+        ipt["batch_text_spans"] = batch_text_spans
+        return ipt
+    prompt_len = len(tokenizer.tokenize(prompt))
+    truncated_texts = [
+        tokenizer.decode(
+            tokenizer.encode(text)[:max_seq_length - prompt_len - 2],
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=True
+        ).strip()
+        for text in texts
+    ]
+    ipt = tokenizer(
+        [prompt + i for i in truncated_texts],
+        padding="longest",
+        truncation=True,
+        max_length=max_seq_length,
+        return_tensors="pt"
+    )
+    batch_text_spans = []
+    for text, data_len in zip(truncated_texts, ipt["attention_mask"].sum(dim=1)):
+        text_spans = [
+            TextSpan(
+                s=0,
+                e=1,
+                module_name="cls_linear",
+            ),
+            TextSpan(
+                s=1 + prompt_len,
+                e=data_len - 1,
+                module_name="chunk_linear",
+            ),
+        ]
+
+        if chunk_size > 1 and chunk_overlap > -1:
+            # chunk_size > 1 means that we need chunk vector
+            if fast_chunk:
+                start_pos, end_pos = 1 + prompt_len, data_len - 1
+                for s in range(start_pos, end_pos, chunk_size):
+                    s -= chunk_overlap
+                    s = max((s, start_pos))
+                    e = min((s + chunk_size, end_pos))
+                    if e - s > 0 and not (s == start_pos and e == end_pos):
+                        text_spans.append(
+                            TextSpan(
+                                s=s,
+                                e=e,
+                                module_name="chunk_linear",
+                            )
+                        )
+
+            else:
+                chunks, chunk_span = recursive_split(text, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+                if len(chunks) > 1:
+                    for (s, e), chunk in zip(chunk_span, chunks):
+                        s = len(tokenizer.tokenize(text[:s])) + 1 + prompt_len
+                        e = len(tokenizer.tokenize(text[:e])) + 1 + prompt_len
+                        if s >= e:
+                            continue
+                        # original chunk vector
+                        text_spans.append(
+                            TextSpan(
+                                s=s,
+                                e=e,
+                                module_name="chunk_linear",
+                                text=chunk
+                            )
+                        )
+
+        batch_text_spans.append(text_spans)
+    ipt["batch_text_spans"] = batch_text_spans
+    return ipt
+
+
+class DeweyV1(ModernBertPreTrainedModel):
+    def __init__(self, config: ModernBertConfig):
+        super().__init__(config)
+        self.config = config
+        self.model = ModernBertModel(config)
+        hidden_size = config.hidden_size
+        vector_size = config.vector_size
+        self.linear_dict = nn.ModuleDict(
+            {
+                "cls_linear": nn.Linear(hidden_size, vector_size, bias=True),
+                "chunk_linear": nn.Linear(hidden_size, vector_size, bias=True),
+            }
+        )
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_multi_vectors(
+            self,
+            batch_token_embeddings: torch.Tensor,
+            batch_text_spans: List[List[TextSpan]],
+            normalize_embeddings: bool = True
+    ) -> List[torch.Tensor]:
+        multi_vectors = []
+        for token_embeddings, text_spans in zip(batch_token_embeddings, batch_text_spans):
+            chunk_vectors = []
+            for text_span in text_spans:
+                s, e = text_span.s, text_span.e
+                if s >= token_embeddings.shape[0] or s >= e:
+                    logging.warning(
+                        f"given span is wrong, s, e, token_embeddings.shape: {s, e, token_embeddings.shape}",
+                    )
+                    s, e = 0, 1
+                mean_tokens_embs = token_embeddings[s:e, :].mean(dim=0, keepdim=True)
+                # if torch.isnan(mean_tokens_embs).any():
+                #     logging.error(f"NaNs in token_embeddings.shape: {token_embeddings.shape},s,e:{s, e}")
+                chunk_vectors.append(
+                    self.linear_dict[text_span.module_name](mean_tokens_embs),
+                )
+            chunk_vectors = torch.cat(chunk_vectors, dim=0)
+            if normalize_embeddings:
+                multi_vectors.append(F.normalize(chunk_vectors, p=2, dim=-1))
+            else:
+                multi_vectors.append(chunk_vectors)
+        return multi_vectors
+
+    def forward(
+            self,
+            input_ids: torch.Tensor,
+            attention_mask: torch.Tensor,
+            batch_text_spans: List[List[TextSpan]],
+            normalize_embeddings: bool = True,
+            *args,
+            **kwargs
+    ) -> List[torch.Tensor]:
+        batch_token_embeddings = self.model(input_ids=input_ids, attention_mask=attention_mask)[0]
+        multi_vectors = self.get_multi_vectors(
+            batch_token_embeddings=batch_token_embeddings,
+            batch_text_spans=batch_text_spans,
+            normalize_embeddings=normalize_embeddings
+        )
+        return multi_vectors
+
+    @torch.no_grad()
+    def encode(
+            self,
+            sentences: str | list[str],
+            batch_size: int = 32,
+            use_cuda: bool = True,
+            show_progress_bar: bool = True,
+            chunk_size: int = 256,
+            chunk_overlap: int = 32,
+            convert_to_tensor: bool = False,
+            max_seq_length: int = 8192,
+            normalize_embeddings: bool = True,
+            prompt: str = "",
+            fast_chunk: bool = False,
+            batch_text_spans: List[List[TextSpan]] = None,
+            *args,
+            **kwargs
+    ) -> Tuple[List[Union[np.ndarray, torch.Tensor]] | torch.Tensor | np.ndarray, List[List[TextSpan]]]:
+        """
+        encode sentences to multi vectors
+        Args:
+            sentences: str | list[str], The sentences to embed
+            batch_size: int
+            use_cuda: bool, Whether to use GPU for inference
+            show_progress_bar: bool, Whether to display the progress bar
+            chunk_size: int, the number tokens of chunk, The recommended size is between 64-1024. The larger the value,
+             the faster the speed, but the effect may decrease. The smaller the value, the slower the speed,
+              and when the value is very small, the effect may also decrease.
+            chunk_overlap: int, Overlap in characters between chunks
+            convert_to_tensor: bool, If true: convert to torch fp32 tensor, otherwise will return fp32 ndarray
+            max_seq_length: int, max length of text
+            normalize_embeddings: bool, whether to do a L2-normalize for vectors
+            prompt: str, the prompt for text, the final text to be encoded is "[CLS]{prompt}{sentence}[SEP]",
+              Note, you CANNOT manually add a prompt before the sentence yourself, as this will affect our length calculation!
+            fast_chunk: bool, if true, directly chunk on input ids, else using RecursiveCharacterTextSplitter
+            batch_text_spans: List[List[TextSpan]], default is None, if provided, the model will not chunk text anymore
+            *args:
+            **kwargs:
+
+        Returns:
+            List[tensor|ndarray], each text's multi vectors
+        """
+        self.eval()
+        # remove duplicate
+        if isinstance(sentences, str):
+            sentences = [sentences]
+        deduplicate_sentences = list(set(sentences))
+        deduplicate_sentences.sort(key=lambda x: len(x), reverse=True)
+        # encode
+        vectors_list, text_spans = [], []
+        for start in tqdm(
+                range(0, len(deduplicate_sentences), batch_size),
+                desc="encoding text...",
+                disable=not show_progress_bar
+        ):
+            batch = deduplicate_sentences[start:start + batch_size]
+            ipt = make_batch_input_for_prediction(
+                batch,
+                tokenizer=self.tokenizer,
+                max_seq_length=max_seq_length,
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                prompt=prompt,
+                fast_chunk=fast_chunk,
+                batch_text_spans=batch_text_spans
+            )
+            text_spans.extend(ipt["batch_text_spans"])
+            ipt = {k: v.cuda() if use_cuda and isinstance(v, torch.Tensor) else v for k, v in ipt.items()}
+            vectors_list.extend(self(**ipt, normalize_embeddings=normalize_embeddings))
+        # print(len(deduplicate_sentences), len(vectors_list), deduplicate_sentences[-1])
+        assert len(deduplicate_sentences) == len(vectors_list)
+        sen2vecs = dict(zip(deduplicate_sentences, vectors_list))
+        sen2spans = dict(zip(deduplicate_sentences, text_spans))
+
+        text_spans = [sen2spans[sen] for sen in sentences]
+        if convert_to_tensor:
+            result = [sen2vecs[sen].cpu().float() for sen in sentences]
+        else:
+            result = [sen2vecs[sen].cpu().float().numpy() for sen in sentences]
+        return result, text_spans

modules.json

@@ -0,0 +1,14 @@
+[
+    {
+        "idx": 0,
+        "name": "0",
+        "path": "",
+        "type": "custom_st.DeweyTransformer"
+    },
+    {
+        "idx": 1,
+        "name": "1",
+        "path": "1_Normalize",
+        "type": "sentence_transformers.models.Normalize"
+    }
+]

sentence_bert_config.json

@@ -0,0 +1,7 @@
+{
+    "max_seq_length": 131072,
+    "do_lower_case": false,
+    "tokenizer_args": {
+        "padding_side": "right"
+    }
+}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,968 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50280": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50281": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50282": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50283": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50284": {
+      "content": "[MASK]",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50285": {
+      "content": "[unused0]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50286": {
+      "content": "[unused1]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "[unused2]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "[unused3]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50289": {
+      "content": "[unused4]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50290": {
+      "content": "[unused5]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50291": {
+      "content": "[unused6]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50292": {
+      "content": "[unused7]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50293": {
+      "content": "[unused8]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50294": {
+      "content": "[unused9]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50295": {
+      "content": "[unused10]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50296": {
+      "content": "[unused11]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50297": {
+      "content": "[unused12]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50298": {
+      "content": "[unused13]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50299": {
+      "content": "[unused14]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50300": {
+      "content": "[unused15]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50301": {
+      "content": "[unused16]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50302": {
+      "content": "[unused17]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50303": {
+      "content": "[unused18]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50304": {
+      "content": "[unused19]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50305": {
+      "content": "[unused20]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50306": {
+      "content": "[unused21]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50307": {
+      "content": "[unused22]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50308": {
+      "content": "[unused23]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50309": {
+      "content": "[unused24]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50310": {
+      "content": "[unused25]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50311": {
+      "content": "[unused26]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50312": {
+      "content": "[unused27]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50313": {
+      "content": "[unused28]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50314": {
+      "content": "[unused29]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50315": {
+      "content": "[unused30]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50316": {
+      "content": "[unused31]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50317": {
+      "content": "[unused32]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50318": {
+      "content": "[unused33]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50319": {
+      "content": "[unused34]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50320": {
+      "content": "[unused35]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50321": {
+      "content": "[unused36]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50322": {
+      "content": "[unused37]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50323": {
+      "content": "[unused38]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50324": {
+      "content": "[unused39]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50325": {
+      "content": "[unused40]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50326": {
+      "content": "[unused41]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50327": {
+      "content": "[unused42]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50328": {
+      "content": "[unused43]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50329": {
+      "content": "[unused44]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50330": {
+      "content": "[unused45]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50331": {
+      "content": "[unused46]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50332": {
+      "content": "[unused47]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50333": {
+      "content": "[unused48]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50334": {
+      "content": "[unused49]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50335": {
+      "content": "[unused50]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50336": {
+      "content": "[unused51]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50337": {
+      "content": "[unused52]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50338": {
+      "content": "[unused53]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50339": {
+      "content": "[unused54]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50340": {
+      "content": "[unused55]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50341": {
+      "content": "[unused56]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50342": {
+      "content": "[unused57]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50343": {
+      "content": "[unused58]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50344": {
+      "content": "[unused59]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50345": {
+      "content": "[unused60]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50346": {
+      "content": "[unused61]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50347": {
+      "content": "[unused62]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50348": {
+      "content": "[unused63]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50349": {
+      "content": "[unused64]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50350": {
+      "content": "[unused65]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50351": {
+      "content": "[unused66]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50352": {
+      "content": "[unused67]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50353": {
+      "content": "[unused68]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50354": {
+      "content": "[unused69]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50355": {
+      "content": "[unused70]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50356": {
+      "content": "[unused71]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50357": {
+      "content": "[unused72]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50358": {
+      "content": "[unused73]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50359": {
+      "content": "[unused74]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50360": {
+      "content": "[unused75]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50361": {
+      "content": "[unused76]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50362": {
+      "content": "[unused77]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50363": {
+      "content": "[unused78]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50364": {
+      "content": "[unused79]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50365": {
+      "content": "[unused80]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50366": {
+      "content": "[unused81]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50367": {
+      "content": "[unused82]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50368": {
+      "content": "<|START_INSTRUCTION|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50369": {
+      "content": "<|END_INSTRUCTION|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "max_length": 8192,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "stride": 0,
+  "tokenizer_class": "PreTrainedTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}