Upload folder using huggingface_hub

README.md

@@ -0,0 +1,237 @@
+---
+library_name: transformers
+base_model:
+- meituan-longcat/LongCat-Flash-Lite
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [meituan-longcat/LongCat-Flash-Lite](https://huggingface.co/meituan-longcat/LongCat-Flash-Lite).
+
+| File path | Size |
+|------|------|
+| model.safetensors | 8.4MB |
+
+
+### Example usage:
+
+```python
+import torch
+import transformers
+
+model_id = "tiny-random/longcat-flash-lite"
+pipe = transformers.pipelines.pipeline(
+    'text-generation',
+    model=model_id,
+    trust_remote_code=True,
+    device_map='cuda',
+    torch_dtype=torch.bfloat16,
+)
+past_key_values = transformers.DynamicCache(config=None)  # set config to None
+r = pipe('Hello, world!', past_key_values=past_key_values, max_new_tokens=32)
+print(r)
+```
+
+### Codes to create this repo:
+
+<details>
+<summary>Python codes</summary>
+
+```python
+import json
+from copy import deepcopy
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    AutoTokenizer,
+    GenerationConfig,
+    set_seed,
+)
+from transformers.models.glm4_moe.modeling_glm4_moe import Glm4MoeRMSNorm
+source_model_id = "meituan-longcat/LongCat-Flash-Lite"
+save_folder = "/tmp/tiny-random/longcat-flash-lite"
+
+Path(save_folder).mkdir(parents=True, exist_ok=True)
+tokenizer = AutoTokenizer.from_pretrained(source_model_id, trust_remote_code=True)
+tokenizer.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json = json.load(f)
+for k, v in config_json['auto_map'].items():
+    config_json['auto_map'][k] = f'{source_model_id}--{v}'
+config_json.update({
+    'num_layers': 2,
+    'hidden_size': 8,
+    'ffn_hidden_size': 32,
+    'expert_ffn_hidden_size': 32,
+    'num_attention_heads': 4,
+    'kv_lora_rank': 384,
+    'n_routed_experts': 32,
+    'q_lora_rank': 32,
+    'qk_nope_head_dim': 64,
+    'qk_rope_head_dim': 192,
+    'head_dim': 192,
+    'qk_head_dim': 256,
+    'v_head_dim': 64,
+    'moe_topk': 12,
+    'zero_expert_num': 16,
+    'emb_split_num': 2,
+    'emb_neighbor_num': 2,
+    'ngram_vocab_size_ratio': 4,
+})
+# del config_json['quantization_config']
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModelForCausalLM.from_config(config, trust_remote_code=True)
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+model = model.cpu()
+# MTP
+model.model.mtp = nn.ModuleDict({
+    "layers": nn.ModuleList([nn.ModuleDict(dict(
+        eh_proj=nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False),
+        enorm=nn.ModuleDict({"m": nn.RMSNorm(config.hidden_size)}),
+        hnorm=nn.ModuleDict({"m": nn.RMSNorm(config.hidden_size)}),
+        input_layernorm=nn.RMSNorm(config.hidden_size),
+        post_attention_layernorm=nn.RMSNorm(config.hidden_size),
+        self_attn=deepcopy(model.model.layers[0].self_attn[0]),
+        transformer_layer=nn.ModuleDict({"mlp": deepcopy(model.model.layers[0].mlps[0])}),
+    ))]),
+    "norm": nn.RMSNorm(config.hidden_size),
+})
+for i in range(config.num_layers):
+    model.model.layers[i].mlp.router = model.model.layers[i].mlp.router.float()
+    # model.model.layers[i].mlp.router.e_score_correction_bias = torch.zeros((config.n_routed_experts + config.zero_expert_num)).float()
+set_seed(42)
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.1)
+        print(name, p.shape, p.dtype)
+model.model.mtp.embed_tokens = deepcopy(model.model.embed_tokens)
+model.model.ngram_embeddings = None  # avoid saving shared params
+
+model.save_pretrained(save_folder)
+torch.set_default_dtype(torch.float32)
+
+print(model.model.rotary_emb.inv_freq.shape)
+# 1 / 0
+# for n, m in model.named_modules():
+#     if 'LongcatFlashMLA' in str(type(m)):
+#         print(n, m.layer_idx)
+
+with open(f"{save_folder}/config.json", "r", encoding='utf-8') as f:
+    config_json = json.load(f)
+    config_json['auto_map'] = {k: v.split('--')[-1] for k, v in config_json['auto_map'].items()}
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+LongcatFlashNgramForCausalLM(
+  (model): LongcatFlashNgramModel(
+    (embed_tokens): Embedding(131072, 8)
+    (layers): ModuleList(
+      (0-1): 2 x LongcatFlashDecoderLayer(
+        (mlp): LongcatFlashMoE(
+          (experts): ModuleList(
+            (0-31): 32 x LongcatFlashMLP(
+              (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+              (up_proj): Linear(in_features=8, out_features=32, bias=False)
+              (down_proj): Linear(in_features=32, out_features=8, bias=False)
+              (act_fn): SiLUActivation()
+            )
+            (32-47): 16 x Identity()
+          )
+          (router): LongcatFlashTopkRouter(
+            (classifier): Linear(in_features=8, out_features=48, bias=False)
+          )
+        )
+        (self_attn): ModuleList(
+          (0-1): 2 x LongcatFlashMLA(
+            (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+            (q_a_layernorm): LongcatFlashRMSNorm((32,), eps=1e-06)
+            (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+            (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+            (kv_a_layernorm): LongcatFlashRMSNorm((384,), eps=1e-06)
+            (kv_b_proj): Linear(in_features=384, out_features=512, bias=False)
+            (o_proj): Linear(in_features=256, out_features=8, bias=False)
+          )
+        )
+        (mlps): ModuleList(
+          (0-1): 2 x LongcatFlashMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): ModuleList(
+          (0-1): 2 x LongcatFlashRMSNorm((8,), eps=1e-05)
+        )
+        (post_attention_layernorm): ModuleList(
+          (0-1): 2 x LongcatFlashRMSNorm((8,), eps=1e-05)
+        )
+      )
+    )
+    (norm): LongcatFlashRMSNorm((8,), eps=1e-05)
+    (rotary_emb): LongcatFlashRotaryEmbedding()
+    (ngram_embeddings): None
+    (mtp): ModuleDict(
+      (layers): ModuleList(
+        (0): ModuleDict(
+          (eh_proj): Linear(in_features=16, out_features=8, bias=False)
+          (enorm): ModuleDict(
+            (m): RMSNorm((8,), eps=None, elementwise_affine=True)
+          )
+          (hnorm): ModuleDict(
+            (m): RMSNorm((8,), eps=None, elementwise_affine=True)
+          )
+          (input_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+          (post_attention_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+          (self_attn): LongcatFlashMLA(
+            (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+            (q_a_layernorm): LongcatFlashRMSNorm((32,), eps=1e-06)
+            (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+            (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+            (kv_a_layernorm): LongcatFlashRMSNorm((384,), eps=1e-06)
+            (kv_b_proj): Linear(in_features=384, out_features=512, bias=False)
+            (o_proj): Linear(in_features=256, out_features=8, bias=False)
+          )
+          (transformer_layer): ModuleDict(
+            (mlp): LongcatFlashMLP(
+              (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+              (up_proj): Linear(in_features=8, out_features=32, bias=False)
+              (down_proj): Linear(in_features=32, out_features=8, bias=False)
+              (act_fn): SiLUActivation()
+            )
+          )
+        )
+      )
+      (norm): RMSNorm((8,), eps=None, elementwise_affine=True)
+      (embed_tokens): Embedding(131072, 8)
+    )
+  )
+  (lm_head): Linear(in_features=8, out_features=131072, bias=False)
+)
+```
+
+</details>

chat_template.jinja

@@ -0,0 +1,81 @@
+{%- set tool_choice = tool_choice | default('auto') %}
+{%- set ns = namespace(tool_types = [], last_query_index = -1) %}
+
+{%- if tools and tool_choice != 'none' %}
+    {{- "<longcat_tool_declare>\n"-}}
+    {{- "# Tools\n" }}
+    {{- "You have access to the following tools:\n\n" }}
+    {%- for tool in tools %}
+        {%- if tool.type not in ns.tool_types %}
+            {%- set ns.tool_types = ns.tool_types + [tool.type] %}
+            {{- "## Tool namespace: " ~ tool.type ~ "\n\n" }}
+        {%- endif %}
+        {%- if tool.type == 'code_interpreter' %}
+            {%- set tool = {"type":"code_interpreter","function":{"name":"code_interpreter_preview","description":"The code will be executed in a stateful Jupyter notebook sandbox environment, only supports local computation, data processing, and file operations.\nCode sandbox environment (network isolated) Any external network requests or online API calls are prohibited.\nIf online functionality is needed, please use other permitted tools.\nCode will respond with the output of the execution or time out after 60.0 seconds. ","parameters":{"type":"object","properties":{"language":{"type":"string","description":"The programming language of the code to be executed. Available values: python (Default), java, go, js, ts, c, c++."},"code":{"type":"string","description":"Python code to be executed must not include the following:\n- Importing network libraries such as requests, httplib, etc.\n- Any form of HTTP requests.\n- External API calls.\n- Network port operations. Example: ```python\nimport pandas as pd\npd.DataFrame({'A':[1,2]})\n```"},"timeout":{"type":"number","description":"The maximum execution time of the code, in seconds. Default is 60.0."}}},"required":["code"]}} %}
+        {%- endif %}
+        {{- "### Tool name: " + tool.function.name + "\n" }}
+        {{- "Description: " + tool.function.description + "\n\n" }}
+        {{- "InputSchema: " + tool.function.parameters | tojson(ensure_ascii=False) + "\n\n" }}
+    {%- endfor %}
+    {{- '**Note**: For each function call, output the function name and arguments within the following XML format:\n<longcat_tool_call>{function-name}\n<longcat_arg_key>{arg-key-1}</longcat_arg_key>\n<longcat_arg_value>{arg-value-1}</longcat_arg_value>\n<longcat_arg_key>{arg-key-2}</longcat_arg_key>\n<longcat_arg_value>{arg-value-2}</longcat_arg_value>\n...\n</longcat_tool_call>\n' }}
+    {{- "</longcat_tool_declare>"-}}
+    {%- for idx in range(messages|length - 1) %}
+        {%- set msg = messages[idx] %}
+        {%- if msg.role == 'assistant' and not msg.tool_calls %}
+            {%- set ns.last_query_index = idx %}
+        {%- endif %}
+    {%- endfor%}
+{%- endif %}
+
+{%- for msg in messages %}
+    {%- if msg.role == "system" %}
+        {{- "<longcat_system>" + msg.content }}
+    {%- elif msg.role == "user" %}
+        {{- "<longcat_user>" }}
+        {%- if msg["files"] %}
+            {{- '<longcat_files>\n' ~ msg.files | tojson(indent=2) ~ '\n</longcat_files>' }}
+        {%- endif %}
+        {{- msg.content }}
+    {%- elif msg.role == "assistant" %}
+        {{- "<longcat_assistant>" }}
+        {%- if enable_thinking == true and msg.reasoning_content and ns.tool_types != [] and loop.index0 > ns.last_query_index %}
+            {{- "\n<longcat_think>\n" ~ msg.reasoning_content ~ "\n</longcat_think>\n" }}
+        {%- endif %}
+        {%- if msg.content%}
+            {{- msg.content }}
+        {%- endif %}
+        {%- if msg.tool_calls %}
+            {%- for tool_call in msg.tool_calls -%}
+                {{- "<longcat_tool_call>" ~ tool_call.function.name ~ "\n" -}}
+                {% set _args = tool_call.function.arguments %}
+                {% for k, v in _args.items() %}
+                    {{- "<longcat_arg_key>" ~ k ~ "</longcat_arg_key>\n" -}}
+                    {{- "<longcat_arg_value>" ~ (v if v is string else v | tojson(ensure_ascii=False)) ~ "</longcat_arg_value>\n" -}}
+                {% endfor %}
+                {{- "</longcat_tool_call>\n" }}
+            {%- endfor %}
+        {%- endif %}
+        {{- "</longcat_s>" -}}
+    {%- elif msg.role == "tool" %}
+        {%- if messages[loop.index0 - 1].role != "tool"%}
+            {{- "<longcat_user>" -}}
+        {%- endif %}
+        {{- "<longcat_tool_response>" ~ msg.content ~ "</longcat_tool_response>"-}}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {%- if enable_thinking == true %}
+        {{- " /think_on" }}
+        {%- if thinking_budget %}
+            {%- if thinking_budget < 1024 %}
+                {%- set thinking_budget = 1024 %}
+            {%- endif%}
+            {{- "\nthinking_budget: < " ~ thinking_budget ~ "."}}
+        {%- endif %}
+        {{- " <longcat_assistant><longcat_think>\n"}}
+    {%- elif enable_thinking == false %}
+        {{- " /think_off <longcat_assistant><longcat_think>\n\n</longcat_think>\n" }}
+    {%- else %}
+        {{- "<longcat_assistant>" }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,58 @@
+{
+  "architectures": [
+    "LongcatFlashNgramForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_longcat_ngram.LongcatFlashNgramConfig",
+    "AutoModel": "modeling_longcat_ngram.LongcatFlashNgramModel",
+    "AutoModelForCausalLM": "modeling_longcat_ngram.LongcatFlashNgramForCausalLM"
+  },
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "emb_neighbor_num": 2,
+  "emb_split_num": 2,
+  "eos_token_id": 2,
+  "expert_ffn_hidden_size": 32,
+  "ffn_hidden_size": 32,
+  "head_dim": 192,
+  "hidden_act": "silu",
+  "hidden_size": 8,
+  "initializer_range": 0.02,
+  "kv_lora_rank": 384,
+  "max_position_embeddings": 327680,
+  "mla_scale_kv_lora": true,
+  "mla_scale_q_lora": true,
+  "model_type": "longcat_flash_ngram",
+  "moe_topk": 12,
+  "n_routed_experts": 32,
+  "ngram_vocab_size_ratio": 4,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 4,
+  "num_layers": 2,
+  "q_lora_rank": 32,
+  "qk_head_dim": 256,
+  "qk_nope_head_dim": 64,
+  "qk_rope_head_dim": 192,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "beta_fast": 32.0,
+    "beta_slow": 1.0,
+    "factor": 10.0,
+    "mscale": 1,
+    "mscale_all_dim": 1,
+    "original_max_position_embeddings": 32768,
+    "rope_type": "yarn"
+  },
+  "rope_theta": 5000000.0,
+  "routed_scaling_factor": 6.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "v_head_dim": 64,
+  "vocab_size": 131072,
+  "zero_expert_num": 16,
+  "zero_expert_type": "identity"
+}

configuration_longcat_ngram.py

@@ -0,0 +1,216 @@
+from transformers.models.longcat_flash import LongcatFlashConfig
+
+
+class LongcatFlashNgramConfig(LongcatFlashConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`LongcatFlashNgramModel`]. It is used to instantiate
+    a LongCat Flash model with N-gram enhanced embeddings according to the specified arguments, defining the model architecture. 
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 131072):
+            Vocabulary size of the LongCat Flash model. Defines the number of different tokens that can be represented by the
+            `input_ids` passed when calling [`LongcatFlashNgramModel`]
+        hidden_size (`int`, *optional*, defaults to 6144):
+            Dimension of the hidden representations.
+        num_hidden_layers (`int`, *optional*, defaults to 56):
+            Number of hidden layers in the Transformer decoder.
+        num_layers (`int`, *optional*, defaults to 28):
+            Number of layers, each with 2 sublayers.
+        num_attention_heads (`int`, *optional*, defaults to 64):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting from a multi-head checkpoint to a GQA checkpoint, each group key and value head should be
+            constructed by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 131072):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon value used by the RMS normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie input and output embeddings.
+        rope_theta (`float`, *optional*, defaults to 10000000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+            `{"type": strategy name, "factor": scaling factor}`.
+        attention_bias (`bool`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        ffn_hidden_size (`int`, *optional*, defaults to 12288):
+            Dimension of the MLP representations.
+        q_lora_rank (`int`, *optional*, defaults to 1536):
+            The rank of the query LoRA projection in MLA (Multi-head Latent Attention).
+        kv_lora_rank (`int`, *optional*, defaults to 512):
+            The rank of the key-value LoRA projection in MLA.
+        qk_nope_head_dim (`int`, *optional*, defaults to 128):
+            The dimension of the non-position encoding part of query/key heads.
+        qk_rope_head_dim (`int`, *optional*, defaults to 64):
+            The dimension of the RoPE part of query/key heads.
+        head_dim (`int`, *optional*, defaults to 64):
+            Standard dimension of qk heads, unused except for CI.
+        v_head_dim (`int`, *optional*, defaults to 128):
+            The dimension of value heads.
+        qk_head_dim (`int`, *optional*):
+            The total dimension of query/key heads. If not specified, set to `qk_nope_head_dim + qk_rope_head_dim`.
+        moe_topk (`int`, *optional*, defaults to 12):
+            Number of experts to route to for each token in the MoE layer.
+        n_routed_experts (`int`, *optional*, defaults to 512):
+            Number of routed experts in the MoE layer.
+        zero_expert_num (`int`, *optional*, defaults to 256):
+            Number of zero experts (identity function) to add to the expert pool.
+        expert_ffn_hidden_size (`int`, *optional*, defaults to 2048):
+            Hidden size of individual expert FFN layers.
+        routed_scaling_factor (`float`, *optional*, defaults to 6.0):
+            Scaling factor applied to the routing weights.
+        emb_neighbor_num (`int`, *optional*):
+            Maximum N-gram length for N-gram embeddings. This parameter determines the context window size for N-gram computation. Higher values capture 
+            longer-range lexical patterns but increase memory usage.
+        emb_split_num (`int`, *optional*):
+            Number of hash functions (or splits) to use for N-gram embeddings. Multiple hash functions help improve the quality of N-gram representations. 
+        ngram_vocab_size_ratio (`float`, *optional*):
+            Ratio multiplier for N-gram vocabulary size relative to the base vocabulary size. The N-gram vocabulary 
+            size is calculated as `vocab_size * ngram_vocab_size_ratio`. 
+
+    Example:
+    ```python
+    >>> from transformers import LongcatFlashNgramModel, LongcatFlashNgramConfig
+
+    >>> # Initializing a LongCat Flash N-gram style configuration
+    >>> configuration = LongcatFlashNgramConfig(
+    ...     emb_neighbor_num=3,
+    ...     emb_split_num=4,
+    ...     ngram_vocab_size_ratio=1.5
+    ... )
+
+    >>> # Initializing a model from the configuration
+    >>> model = LongcatFlashNgramModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "longcat_flash_ngram"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    base_model_tp_plan = {
+        "layers.*.self_attn.*.q_b_proj": "colwise",
+        "layers.*.self_attn.*.kv_b_proj": "colwise",
+        "layers.*.self_attn.*.o_proj": "rowwise",
+        "layers.*.mlps.*.gate_proj": "colwise",
+        "layers.*.mlps.*.up_proj": "colwise",
+        "layers.*.mlps.*.down_proj": "rowwise",
+        "layers.*.mlp.experts.*.gate_proj": "colwise",
+        "layers.*.mlp.experts.*.up_proj": "colwise",
+        "layers.*.mlp.experts.*.down_proj": "rowwise",
+    }
+
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    def __init__(
+        self,
+        vocab_size=131072,
+        hidden_size=6144,
+        num_hidden_layers=56,
+        num_layers=28,
+        num_attention_heads=64,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        max_position_embeddings=131072,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        rope_theta=10000000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        ffn_hidden_size=12288,
+        q_lora_rank=1536,
+        kv_lora_rank=512,
+        qk_nope_head_dim=128,
+        qk_rope_head_dim=64,
+        head_dim=64,
+        v_head_dim=128,
+        qk_head_dim=None,
+        moe_topk=12,
+        n_routed_experts=512,
+        zero_expert_num=256,
+        expert_ffn_hidden_size=2048,
+        routed_scaling_factor=6.0,
+        emb_neighbor_num=None,
+        emb_split_num=None,
+        ngram_vocab_size_ratio=None,
+        **kwargs,
+    ):
+        # N-gram embedding specific parameters
+        self.emb_neighbor_num = emb_neighbor_num
+        self.emb_split_num = emb_split_num
+        self.ngram_vocab_size_ratio = ngram_vocab_size_ratio
+
+        super().__init__(
+            vocab_size=vocab_size,
+            hidden_size=hidden_size,
+            num_hidden_layers=num_hidden_layers,
+            num_layers=num_layers,
+            num_attention_heads=num_attention_heads,
+            num_key_value_heads=num_key_value_heads,
+            hidden_act=hidden_act,
+            max_position_embeddings=max_position_embeddings,
+            initializer_range=initializer_range,
+            rms_norm_eps=rms_norm_eps,
+            use_cache=use_cache,
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            rope_theta=rope_theta,
+            rope_scaling=rope_scaling,
+            attention_bias=attention_bias,
+            attention_dropout=attention_dropout,
+            ffn_hidden_size=ffn_hidden_size,
+            q_lora_rank=q_lora_rank,
+            kv_lora_rank=kv_lora_rank,
+            qk_nope_head_dim=qk_nope_head_dim,
+            qk_rope_head_dim=qk_rope_head_dim,
+            head_dim=head_dim,
+            v_head_dim=v_head_dim,
+            qk_head_dim=qk_head_dim,
+            moe_topk=moe_topk,
+            n_routed_experts=n_routed_experts,
+            zero_expert_num=zero_expert_num,
+            expert_ffn_hidden_size=expert_ffn_hidden_size,
+            routed_scaling_factor=routed_scaling_factor,
+            **kwargs,
+        )
+
+
+__all__ = ["LongcatFlashNgramConfig"]

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "transformers_version": "4.57.6"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b07278fd0e88b73482a5ad791bdc6f68d086050dae57289ad19f2f9b48539848
+size 8798040

modeling_longcat_ngram.py

@@ -0,0 +1,338 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 Meituan
+# This code is licensed under the MIT License, for details, see the ./LICENSE file. 
+
+from typing import Optional, Tuple, Dict, List
+
+import torch
+from torch import nn
+
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.masking_utils import create_causal_mask
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.processing_utils import Unpack
+from transformers.utils import auto_docstring, logging
+from transformers.models.longcat_flash.modeling_longcat_flash import (
+    LongcatFlashForCausalLM,
+    LongcatFlashModel,
+    LongcatFlashRMSNorm,
+    LongcatFlashRotaryEmbedding,
+    LongcatFlashDecoderLayer,
+    LongcatFlashPreTrainedModel,
+)
+from .configuration_longcat_ngram import LongcatFlashNgramConfig
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring
+class LongcatFlashNgramPreTrainedModel(LongcatFlashPreTrainedModel):
+    pass
+
+
+class NgramCache(DynamicCache):
+    """
+    Extended DynamicCache for storing N-gram context alongside KV cache.
+    """
+    def __init__(self, config=None):
+        super().__init__()
+        self.ngram_context = None
+        # Keep only n-1 tokens (minimum needed for N-gram computation)
+        self.max_context_len = config.emb_neighbor_num - 1
+    
+    def update_ngram_context(self, new_tokens: torch.Tensor) -> None:
+        """
+        Update N-gram context with window management.
+        
+        Args:
+            new_tokens: New tokens to append, shape (batch_size, seq_len)
+        """
+        if self.ngram_context is None:
+            self.ngram_context = new_tokens.clone()
+        else:
+            self.ngram_context = torch.cat([self.ngram_context, new_tokens], dim=-1)
+        
+        # Truncate to maintain constant memory footprint
+        if self.ngram_context.size(-1) > self.max_context_len:
+            self.ngram_context = self.ngram_context[..., -self.max_context_len:]
+    
+    def reorder_cache(self, beam_idx: torch.LongTensor) -> "Cache":
+        """Reorder cache for beam search."""
+        # Reorder parent's KV cache
+        super().reorder_cache(beam_idx)
+        
+        # Reorder N-gram context
+        if self.ngram_context is not None:
+            self.ngram_context = self.ngram_context.index_select(0, beam_idx.to(self.ngram_context.device))
+        
+        return self
+
+
+class NgramEmbedding(nn.Module):
+    """
+    Computes embeddings enriched with N-gram features without maintaining internal state.
+    """
+    def __init__(self, config, base_embeddings):
+        super().__init__()
+        self.config = config
+        self.word_embeddings = base_embeddings
+        
+        self.m = config.ngram_vocab_size_ratio * config.vocab_size
+        self.k = config.emb_split_num
+        self.n = config.emb_neighbor_num
+        
+        self._init_ngram_embeddings()
+        self._vocab_mods_cache = None
+        
+    def _init_ngram_embeddings(self) -> None:
+        """Initialize N-gram embedding and projection layers."""
+        num_embedders = self.k * (self.n - 1)
+        emb_dim = self.config.hidden_size // num_embedders
+        
+        embedders = []
+        post_projs = []
+        
+        for i in range(num_embedders):
+            vocab_size = int(self.m + i * 2 + 1)
+            emb = nn.Embedding(vocab_size, emb_dim, padding_idx=self.config.pad_token_id)
+            proj = nn.Linear(emb_dim, self.config.hidden_size, bias=False)
+            embedders.append(emb)
+            post_projs.append(proj)
+        
+        self.embedders = nn.ModuleList(embedders)
+        self.post_projs = nn.ModuleList(post_projs)
+    
+    def _shift_right_ignore_eos(self, tensor: torch.Tensor, n: int, eos_token_id: int = 2) -> torch.Tensor:
+        """Shift tensor right by n positions, resetting at EOS tokens."""
+        batch_size, seq_len = tensor.shape
+        result = torch.zeros_like(tensor)
+        eos_mask = (tensor == eos_token_id)
+        
+        for i in range(batch_size):
+            eos_positions = eos_mask[i].nonzero(as_tuple=True)[0]
+            prev_idx = 0
+            
+            for eos_idx in eos_positions:
+                end_idx = eos_idx.item() + 1
+                if end_idx - prev_idx > n:
+                    result[i, prev_idx+n:end_idx] = tensor[i, prev_idx:end_idx-n]
+                prev_idx = end_idx
+            
+            if prev_idx < seq_len and seq_len - prev_idx > n:
+                result[i, prev_idx+n:seq_len] = tensor[i, prev_idx:seq_len-n]
+        
+        return result
+    
+    def _precompute_vocab_mods(self) -> Dict[Tuple[int, int], List[int]]:
+        """Precompute modular arithmetic values for vocabulary."""
+        if self._vocab_mods_cache is not None:
+            return self._vocab_mods_cache
+        
+        vocab_mods = {}
+        vocab_size = self.config.vocab_size
+        
+        for i in range(2, self.n + 1):
+            for j in range(self.k):
+                index = (i - 2) * self.k + j
+                emb_vocab_dim = int(self.m + index * 2 + 1)
+                
+                mods = []
+                power_mod = 1
+                for _ in range(i - 1):
+                    power_mod = (power_mod * vocab_size) % emb_vocab_dim
+                    mods.append(power_mod)
+                
+                vocab_mods[(i, j)] = mods
+        
+        self._vocab_mods_cache = vocab_mods
+        return vocab_mods
+    
+    def _get_ngram_ids(
+        self, 
+        input_ids: torch.Tensor, 
+        shifted_ids: Dict[int, torch.Tensor], 
+        vocab_mods: List[int], 
+        ngram: int
+    ) -> torch.Tensor:
+        """Compute N-gram hash IDs using polynomial rolling hash."""
+        ngram_ids = input_ids.clone()
+        for k in range(2, ngram + 1):
+            ngram_ids = ngram_ids + shifted_ids[k] * vocab_mods[k - 2]
+        return ngram_ids
+    
+    def forward(
+        self, 
+        input_ids: torch.Tensor, 
+        ngram_context: Optional[torch.Tensor] = None
+    ) -> torch.Tensor:
+        """
+        Stateless forward pass.
+        
+        Args:
+            input_ids: Current input token IDs of shape (batch_size, seq_len)
+            ngram_context: Optional historical context of shape (batch_size, context_len)
+            
+        Returns:
+            Embedding tensor of shape (batch_size, seq_len, hidden_size)
+        """
+        seq_len = input_ids.size(-1)
+        
+        # Determine complete context
+        if ngram_context is not None:
+            context = torch.cat([ngram_context[..., -(self.n-1):], input_ids], dim=-1)
+        else:
+            context = input_ids
+        
+        # Base word embeddings
+        device = self.word_embeddings.weight.device
+        x = self.word_embeddings(input_ids.to(device)).clone()
+        
+        # Precompute modular values
+        vocab_mods = self._precompute_vocab_mods()
+        
+        # Compute shifted IDs
+        shifted_ids = {}
+        for i in range(2, self.n + 1):
+            shifted_ids[i] = self._shift_right_ignore_eos(
+                context, i - 1, eos_token_id=self.config.eos_token_id
+            )
+        
+        # Add N-gram embeddings
+        for i in range(2, self.n + 1):
+            for j in range(self.k):
+                index = (i - 2) * self.k + j
+                emb_vocab_dim = int(self.m + index * 2 + 1)
+                
+                ngram_ids = self._get_ngram_ids(context, shifted_ids, vocab_mods[(i, j)], ngram=i)
+                new_ids = (ngram_ids % emb_vocab_dim)[..., -seq_len:]
+                
+                embedder_device = self.embedders[index].weight.device
+                x_ngram = self.embedders[index](new_ids.to(embedder_device))
+                
+                proj_device = self.post_projs[index].weight.device
+                x_proj = self.post_projs[index](x_ngram.to(proj_device))
+                x = x + x_proj.to(x.device)
+        
+        # Normalize
+        x = x / (1 + self.k * (self.n - 1))
+        
+        return x
+
+
+class LongcatFlashNgramModel(LongcatFlashModel):
+    """LongcatFlash model with N-gram enhanced embeddings."""
+    _keys_to_ignore_on_load_unexpected = [r"model\.mtp.*"]
+    config_class = LongcatFlashNgramConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.ngram_embeddings = NgramEmbedding(config, self.embed_tokens)
+
+        self.layers = nn.ModuleList(
+            [LongcatFlashDecoderLayer(config, layer_idx) for layer_idx in range(config.num_layers)]
+        )
+        
+        self.head_dim = config.head_dim  
+        self.config.num_hidden_layers = 2 * config.num_layers
+        self.norm = LongcatFlashRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = LongcatFlashRotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        **kwargs
+    ) -> BaseModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        # Extract N-gram context if available
+        ngram_context = None
+        if isinstance(past_key_values, NgramCache) and past_key_values.ngram_context is not None:
+            ngram_context = past_key_values.ngram_context
+
+        if inputs_embeds is None:
+            inputs_embeds = self.ngram_embeddings(input_ids, ngram_context=ngram_context)
+
+        # Initialize NgramCache if needed
+        if use_cache and past_key_values is None:
+            past_key_values = NgramCache(config=self.config)
+        
+        # Update N-gram context
+        if use_cache and isinstance(past_key_values, NgramCache):
+            past_key_values.update_ngram_context(input_ids)
+
+        # Prepare cache position
+        if cache_position is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            cache_position = torch.arange(
+                inputs_embeds.shape[1], device=inputs_embeds.device
+            ) + past_seen_tokens
+
+        if position_ids is None:
+            position_ids = cache_position.unsqueeze(0)
+
+        # Create causal mask
+        causal_mask = create_causal_mask(
+            config=self.config,
+            input_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            cache_position=cache_position,
+            past_key_values=past_key_values,
+            position_ids=position_ids,
+        )
+
+        # Forward through decoder layers
+        hidden_states = inputs_embeds
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        for decoder_layer in self.layers[: self.config.num_layers]:
+            hidden_states = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                cache_position=cache_position,
+                position_embeddings=position_embeddings,
+                **kwargs,
+            )
+
+        hidden_states = self.norm(hidden_states)
+        
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values,
+            hidden_states=None,
+            attentions=None,
+        )
+
+
+class LongcatFlashNgramForCausalLM(LongcatFlashForCausalLM):
+    """LongcatFlash model for causal language modeling with N-gram embeddings."""
+    _keys_to_ignore_on_load_unexpected = [r"model\.mtp.*"]
+    config_class = LongcatFlashNgramConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = LongcatFlashNgramModel(config)
+
+    @torch.no_grad()
+    def generate(self, inputs=None, generation_config=None, **kwargs):
+        """Override to ensure NgramCache is used."""
+
+        if "past_key_values" not in kwargs or kwargs["past_key_values"] is None:
+            kwargs["past_key_values"] = NgramCache(config=self.config)
+        
+        return super().generate(inputs=inputs, generation_config=generation_config, **kwargs)
+
+__all__ = ["LongcatFlashNgramPreTrainedModel", "LongcatFlashNgramModel", "LongcatFlashNgramForCausalLM"]

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<longcat_s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</longcat_s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<longcat_pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<longcat_unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,1810 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": true,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<longcat_unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<longcat_s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</longcat_s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<longcat_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<shift_unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<shift_s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "</shift_s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<shift_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<mask_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<commit_before>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<commit_msg>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<commit_after>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<program_lang>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<|image_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<|url_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<|hyperlink_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<|table_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<|equation_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<|code_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<|reference_placeholder|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<longcat_think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "</longcat_think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<longcat_answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "</longcat_answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<longcat_files>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "</longcat_files>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<longcat_tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "</longcat_tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<longcat_tool_declare>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "45": {
+      "content": "</longcat_tool_declare>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46": {
+      "content": "<longcat_system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "47": {
+      "content": "<longcat_user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48": {
+      "content": "<longcat_assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49": {
+      "content": "<longcat_tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "</longcat_tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<longcat_arg_key>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "</longcat_arg_key>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<longcat_arg_value>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "</longcat_arg_value>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<mask_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "56": {
+      "content": "<mask_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "57": {
+      "content": "<mask_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "58": {
+      "content": "<mask_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "59": {
+      "content": "<mask_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "60": {
+      "content": "<mask_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "61": {
+      "content": "<mask_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "62": {
+      "content": "<mask_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "63": {
+      "content": "<mask_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64": {
+      "content": "<mask_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65": {
+      "content": "<mask_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "66": {
+      "content": "<mask_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "67": {
+      "content": "<mask_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "68": {
+      "content": "<mask_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "69": {
+      "content": "<mask_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "70": {
+      "content": "<mask_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "71": {
+      "content": "<mask_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "72": {
+      "content": "<mask_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73": {
+      "content": "<mask_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "74": {
+      "content": "<mask_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "75": {
+      "content": "<mask_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "76": {
+      "content": "<mask_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "77": {
+      "content": "<mask_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "78": {
+      "content": "<mask_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "79": {
+      "content": "<mask_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "80": {
+      "content": "<mask_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "81": {
+      "content": "<mask_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "82": {
+      "content": "<mask_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "83": {
+      "content": "<mask_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "84": {
+      "content": "<mask_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "85": {
+      "content": "<mask_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "86": {
+      "content": "<mask_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "87": {
+      "content": "<mask_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "88": {
+      "content": "<mask_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "89": {
+      "content": "<mask_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "90": {
+      "content": "<mask_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "91": {
+      "content": "<mask_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92": {
+      "content": "<mask_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "93": {
+      "content": "<mask_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "94": {
+      "content": "<mask_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "95": {
+      "content": "<mask_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96": {
+      "content": "<mask_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "97": {
+      "content": "<mask_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "98": {
+      "content": "<mask_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "99": {
+      "content": "<mask_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "<mask_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "<mask_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "<mask_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "<mask_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "<mask_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "105": {
+      "content": "<mask_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<mask_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<mask_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "<mask_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "109": {
+      "content": "<mask_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110": {
+      "content": "<mask_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "111": {
+      "content": "<mask_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "112": {
+      "content": "<mask_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "113": {
+      "content": "<mask_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "114": {
+      "content": "<mask_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "115": {
+      "content": "<mask_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "116": {
+      "content": "<mask_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "117": {
+      "content": "<mask_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "118": {
+      "content": "<mask_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "119": {
+      "content": "<mask_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "120": {
+      "content": "<mask_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "121": {
+      "content": "<mask_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "122": {
+      "content": "<mask_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "123": {
+      "content": "<mask_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "124": {
+      "content": "<mask_100>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "125": {
+      "content": "<mask_101>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "126": {
+      "content": "<mask_102>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "127": {
+      "content": "<mask_103>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128": {
+      "content": "<mask_104>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "129": {
+      "content": "<mask_105>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "130": {
+      "content": "<mask_106>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "131": {
+      "content": "<mask_107>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "132": {
+      "content": "<mask_108>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "133": {
+      "content": "<mask_109>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "134": {
+      "content": "<mask_110>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "135": {
+      "content": "<mask_111>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "136": {
+      "content": "<mask_112>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "137": {
+      "content": "<mask_113>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "138": {
+      "content": "<mask_114>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "139": {
+      "content": "<mask_115>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "140": {
+      "content": "<mask_116>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "141": {
+      "content": "<mask_117>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "142": {
+      "content": "<mask_118>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "143": {
+      "content": "<mask_119>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "144": {
+      "content": "<mask_120>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145": {
+      "content": "<mask_121>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "146": {
+      "content": "<mask_122>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "147": {
+      "content": "<mask_123>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "148": {
+      "content": "<mask_124>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "149": {
+      "content": "<mask_125>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "150": {
+      "content": "<mask_126>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151": {
+      "content": "<mask_127>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152": {
+      "content": "<mask_128>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "153": {
+      "content": "<mask_129>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "154": {
+      "content": "<mask_130>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "155": {
+      "content": "<mask_131>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156": {
+      "content": "<mask_132>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157": {
+      "content": "<mask_133>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "158": {
+      "content": "<mask_134>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "159": {
+      "content": "<mask_135>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "160": {
+      "content": "<mask_136>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "161": {
+      "content": "<mask_137>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "162": {
+      "content": "<mask_138>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "163": {
+      "content": "<mask_139>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "164": {
+      "content": "<mask_140>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "165": {
+      "content": "<mask_141>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "166": {
+      "content": "<mask_142>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "167": {
+      "content": "<mask_143>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "168": {
+      "content": "<mask_144>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "169": {
+      "content": "<mask_145>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "170": {
+      "content": "<mask_146>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "171": {
+      "content": "<mask_147>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "172": {
+      "content": "<mask_148>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "173": {
+      "content": "<mask_149>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "174": {
+      "content": "<mask_150>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "175": {
+      "content": "<mask_151>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "176": {
+      "content": "<mask_152>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "177": {
+      "content": "<mask_153>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "178": {
+      "content": "<mask_154>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "179": {
+      "content": "<mask_155>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "180": {
+      "content": "<mask_156>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "181": {
+      "content": "<mask_157>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "182": {
+      "content": "<mask_158>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "183": {
+      "content": "<mask_159>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "184": {
+      "content": "<mask_160>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "185": {
+      "content": "<mask_161>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "186": {
+      "content": "<mask_162>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "187": {
+      "content": "<mask_163>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "188": {
+      "content": "<mask_164>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "189": {
+      "content": "<mask_165>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "190": {
+      "content": "<mask_166>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "191": {
+      "content": "<mask_167>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "192": {
+      "content": "<mask_168>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "193": {
+      "content": "<mask_169>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "194": {
+      "content": "<mask_170>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "195": {
+      "content": "<mask_171>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "196": {
+      "content": "<mask_172>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "197": {
+      "content": "<mask_173>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "198": {
+      "content": "<mask_174>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "199": {
+      "content": "<mask_175>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200": {
+      "content": "<mask_176>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "201": {
+      "content": "<mask_177>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "202": {
+      "content": "<mask_178>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "203": {
+      "content": "<mask_179>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "204": {
+      "content": "<mask_180>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "205": {
+      "content": "<mask_181>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "206": {
+      "content": "<mask_182>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "207": {
+      "content": "<mask_183>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "208": {
+      "content": "<mask_184>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "209": {
+      "content": "<mask_185>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "210": {
+      "content": "<mask_186>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "211": {
+      "content": "<mask_187>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "212": {
+      "content": "<mask_188>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "213": {
+      "content": "<mask_189>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "214": {
+      "content": "<mask_190>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "215": {
+      "content": "<mask_191>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "216": {
+      "content": "<mask_192>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "217": {
+      "content": "<mask_193>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "218": {
+      "content": "<mask_194>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "219": {
+      "content": "<mask_195>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "220": {
+      "content": "<mask_196>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "221": {
+      "content": "<mask_197>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "222": {
+      "content": "<mask_198>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "223": {
+      "content": "<mask_199>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<longcat_s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</longcat_s>",
+  "extra_special_tokens": {},
+  "merges_file": null,
+  "model_max_length": 131072,
+  "pad_token": "<longcat_pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<longcat_unk>",
+  "vocab_file": null
+}