Upload folder using huggingface_hub

__init__.py

@@ -0,0 +1,7 @@
+from .configuration_fastslm import FastSLMConfig
+from .modeling_fastslm import FastSLMForConditionalGeneration # FastALMForCausalLM
+
+from transformers import AutoConfig, AutoModelForCausalLM
+
+AutoConfig.register("fastslm", FastSLMConfig)
+AutoModelForCausalLM.register(FastSLMConfig, FastSLMForConditionalGeneration)

added_tokens.json

@@ -0,0 +1,37 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|ASR|>": 151674,
+  "<|AST|>": 151675,
+  "<|AUDIO|>": 151669,
+  "<|EN|>": 151672,
+  "<|KO|>": 151673,
+  "<|SQQA|>": 151677,
+  "<|SSUM|>": 151676,
+  "<|audio_bos|>": 151670,
+  "<|audio_eos|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,102 @@
+{
+  "architectures": [
+    "FastSLMForConditionalGeneration"
+  ],
+  "encoder_config": {
+    "compression_size": 50,
+    "model_type": "fastslm_speech_encoder",
+    "n_ctx": 1500,
+    "n_head": 20,
+    "n_layer": 32,
+    "n_mels": 128,
+    "n_state": 1280,
+    "stage_tokens": [
+      80,
+      80,
+      80
+    ]
+  },
+  "llm_config": {
+    "_name_or_path": "Qwen/Qwen3-4B",
+    "architectures": [
+      "Qwen3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2560,
+    "initializer_range": 0.02,
+    "intermediate_size": 9728,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 40960,
+    "max_window_layers": 36,
+    "model_type": "qwen3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "llm_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "lora_a": 64,
+  "lora_r": 16,
+  "low_resource": false,
+  "model_type": "fastslm",
+  "torch_dtype": "float32",
+  "transformers_version": "4.53.1"
+}

configuration_fastslm.py

@@ -0,0 +1,63 @@
+from transformers import PretrainedConfig, AutoConfig
+
+class FastSLMSpeechEncoderConfig(PretrainedConfig):
+    model_type = "fastslm_speech_encoder"
+    def __init__(
+        self,
+        n_mels=128,
+        n_ctx=1500,
+        n_state=1280,
+        n_head=20,
+        n_layer=32,
+        stage_tokens=[80, 80, 80],
+        compression_size=50,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.n_mels = n_mels
+        self.n_ctx = n_ctx
+        self.n_state = n_state
+        self.n_head = n_head
+        self.n_layer = n_layer
+        self.stage_tokens = stage_tokens
+        self.compression_size = compression_size
+
+class FastSLMConfig(PretrainedConfig):
+    model_type = "fastslm"
+    def __init__(
+        self,
+        encoder_config=None,
+        llm_config=None,
+        lora_r=16,
+        lora_a=64,
+        llm_modules=None,
+        low_resource=False,
+        **kwargs
+    ):
+        # llm_modules 기본값
+        if llm_modules is None:
+            llm_modules = ["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"]
+        
+        # LLM config 처리: dict면 AutoConfig로 변환
+        if llm_config is None:
+            llm_config = AutoConfig.from_pretrained("Qwen/Qwen3-4B")
+        elif isinstance(llm_config, dict):
+            if "_name_or_path" in llm_config:
+                llm_config = AutoConfig.from_pretrained(llm_config["_name_or_path"], **llm_config)
+            else:
+                llm_config = AutoConfig.from_dict(llm_config)
+        
+        # Encoder config 처리
+        if encoder_config is None:
+            encoder_config = FastSLMSpeechEncoderConfig()
+        elif isinstance(encoder_config, dict):
+            encoder_config = FastSLMSpeechEncoderConfig(**encoder_config)
+
+        self.llm_config = llm_config
+        self.encoder_config = encoder_config
+        self.lora_r = lora_r
+        self.lora_a = lora_a
+        self.llm_modules = llm_modules
+        self.low_resource = low_resource
+
+        super().__init__(**kwargs)

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.53.1"
+}

llm/README.md

@@ -0,0 +1,202 @@
+---
+base_model: Qwen/Qwen3-4B
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

llm/adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-4B",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.01,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "gate_proj",
+    "up_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llm/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b3c7762f65f50c847fdfd682096684294f02093c3c7658ade4181b7fda56dc
+size 132187888

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59b9453b73e8d8b5f3e806b022b2661d2a44b0db781dfe5182bcb33a7181cc4
+size 4954711528

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6158e0d84272203c637d822df06cf4a8f25ef42b5159e454a6e0dc4c40731f35
+size 4983450856

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90f7bcf65c37169472812a4b6860ecb9d496c0b11576063d1ebd714d7b9837f9
+size 1805593888

modeling_fastslm.py

@@ -0,0 +1,453 @@
+# FastSLM/modeling_fastslm.py
+
+import torch
+import torch.nn as nn
+import torchaudio
+import torch.nn.functional as F
+import numpy as np
+from torch import Tensor
+import whisper
+from einops import rearrange
+from typing import Optional, List
+
+from peft import (
+    LoraConfig, 
+    get_peft_model
+)
+from transformers import (
+    AutoModelForCausalLM, 
+    AutoTokenizer, 
+    PreTrainedModel, 
+    GenerationMixin,
+    AutoConfig
+    )
+from .modeling_whisper import AudioEncoder
+from .configuration_fastslm import FastSLMConfig
+# Check for scaled_dot_product_attention availability
+try:
+    from torch.nn.functional import scaled_dot_product_attention
+    SDPA_AVAILABLE = True
+except (ImportError, RuntimeError, OSError):
+    scaled_dot_product_attention = None
+    SDPA_AVAILABLE = False
+
+LANGUAGES = {
+    "en": "english",
+    "ko": "korean"
+}
+
+def set_trainable_parameters(module, requires_grad=False):
+    for param in module.parameters():
+        param.requires_grad = requires_grad
+    module._requires_grad = requires_grad
+
+# --- Helper Modules (Compressor, MHSA, Attention, Downsampler) ---
+
+class Compressor(nn.Module):
+    def __init__(self, embed_dim, num_heads, num_query, n_ctx):
+        super().__init__()
+        self.num_heads = num_heads
+        self.head_dims = embed_dim // num_heads
+        self.n_ctx = n_ctx
+        
+        self.query = nn.Parameter(torch.randn(1, num_query, embed_dim))
+        nn.init.normal_(self.query, mean=0.0, std=0.02)
+        
+        self.q_ln = nn.LayerNorm(embed_dim, eps=1e-5)
+        self.kv_ln = nn.LayerNorm(embed_dim, eps=1e-5)
+        
+        self.kv_proj = nn.Identity()
+        self.out_proj = nn.Linear(embed_dim, embed_dim)
+
+        self.register_buffer("q_pos_embeds", self.sinusoids(num_query, embed_dim))
+        self.register_buffer("kv_pos_embeds", self.sinusoids(n_ctx, embed_dim))
+        
+        self.init_weights()
+        
+    def init_weights(self):
+        nn.init.constant_(self.q_ln.bias, 0)
+        nn.init.constant_(self.q_ln.weight, 1.0)
+        nn.init.constant_(self.kv_ln.bias, 0)
+        nn.init.constant_(self.kv_ln.weight, 1.0)
+    
+    def sinusoids(self, length, channels, max_timescale=10000):
+        assert channels % 2 == 0
+        log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
+        inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
+        scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
+        return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
+
+    def forward(self, x: Tensor):
+        q = self.q_ln(self.query.to(x.device))
+        x = self.kv_ln(self.kv_proj(x))
+
+        q = rearrange(q + self.q_pos_embeds, 'b l (h d) -> b h l d', h=self.num_heads, d=self.head_dims)
+        k = rearrange(x + self.kv_pos_embeds, 'b l (h d) -> b h l d', h=self.num_heads, d=self.head_dims)
+        v = rearrange(x, 'b l (h d) -> b h l d', h=self.num_heads, d=self.head_dims)
+
+        attn = scaled_dot_product_attention(q, k, v)
+        attn = rearrange(attn, 'b h l d -> b l (h d)')
+        x = self.out_proj(attn)
+        return x
+
+class MHSA(nn.Module):
+    def __init__(self, embed_dim, num_heads):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.head_dims = embed_dim // num_heads
+        self.q = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.k = nn.Linear(embed_dim, embed_dim, bias=False)
+        self.v = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+    
+    def forward(self, x, xa=None, mask=None):
+        q = self.q(x)
+        k = self.k(x if xa is None else xa)
+        v = self.v(x if xa is None else xa)
+        
+        q = rearrange(q, 'b l (h d) -> b h l d', h=self.num_heads, d=self.head_dims)
+        k = rearrange(k, 'b l (h d) -> b h l d', h=self.num_heads, d=self.head_dims)
+        v = rearrange(v, 'b l (h d) -> b h l d', h=self.num_heads, d=self.head_dims)
+
+        attn = scaled_dot_product_attention(q, k, v, is_causal=mask is not None)
+        attn = rearrange(attn, 'b h l d -> b l (h d)')
+        
+        out = self.out_proj(attn)
+        return out
+
+class Attention(nn.Module):
+    def __init__(self, embed_dim, num_heads):
+        super().__init__()
+        self.attn = MHSA(embed_dim=embed_dim, num_heads=num_heads)
+        self.cross_attn = MHSA(embed_dim=embed_dim, num_heads=num_heads)
+        self.norm1 = nn.LayerNorm(embed_dim, eps=1e-5)
+        self.norm2 = nn.LayerNorm(embed_dim, eps=1e-5)
+
+    def forward(self, x: Tensor, xa: Optional[Tensor] = None):
+        x = x + self.attn(self.norm1(x))
+        x = x + self.cross_attn(x=self.norm2(x), xa=xa)
+        return x
+
+class Downsampler(nn.Module):
+    def __init__(self, embed_dim: int):
+        super().__init__()
+        self.conv1 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, padding=1)
+        self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1)
+        self.ln_post = nn.LayerNorm(embed_dim, eps=1e-5)
+
+    def forward(self, x: Tensor):
+        x = F.gelu(self.conv1(x))
+        x = F.gelu(self.conv2(x))
+        x = x.permute(0, 2, 1)
+        x = self.ln_post(x)
+        return x
+
+# --- Speech Encoder Module ---
+
+class SpeechEncoder(nn.Module):
+    def __init__(self, config: FastSLMConfig):
+        super().__init__()
+        # Initialize the Whisper encoder from its specific sub-configuration
+        self._device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.whisper = AudioEncoder(
+            n_mels=config.encoder_config.n_mels,
+            n_ctx=config.encoder_config.n_ctx,
+            n_state=config.encoder_config.n_state,
+            n_head=config.encoder_config.n_head,
+            n_layer=config.encoder_config.n_layer
+        )
+        self.n_mels = config.encoder_config.n_mels
+        # Freeze the Whisper encoder as it's not trained
+        for param in self.whisper.parameters():
+            param.requires_grad = False
+            
+        # Initialize the projection layer to match the LLM's hidden dimension
+        self.llm_proj = nn.Linear(config.encoder_config.n_state, config.llm_config.hidden_size)
+
+        # Initialize the hierarchical compressors using parameters from the config
+        num_heads = config.encoder_config.n_head
+        stage_tokens = config.encoder_config.stage_tokens
+        self.compression_size = config.encoder_config.compression_size
+        self.n_state = config.encoder_config.n_state
+        self.low_resource = config.low_resource
+
+        self.compressor1 = Compressor(config.encoder_config.n_state, num_heads, stage_tokens[0], 1500)
+        self.stage1 = Downsampler(config.encoder_config.n_state)
+        self.compressor2 = Compressor(config.encoder_config.n_state, num_heads, stage_tokens[1], 750)
+        self.stage2 = Downsampler(config.encoder_config.n_state)
+        self.compressor3 = Compressor(config.encoder_config.n_state, num_heads, stage_tokens[2], 375)
+        self.compressor = Compressor(config.encoder_config.n_state, num_heads, self.compression_size, sum(stage_tokens))
+        
+        self.out_attn = nn.ModuleList([
+            Attention(config.encoder_config.n_state, num_heads) for _ in range(2)
+        ])
+
+    def embed_audio(self, mel: torch.Tensor):
+        output = self.whisper(mel)
+        # return output.last_hidden_state
+        return output
+    
+    def forward(self, wav_list: List[torch.Tensor]):
+        if len(wav_list) <= 1:  
+            speech_features = self.process_audio_for_llm_input(wav_list)
+            speech_attn_mask = torch.zeros(1,speech_features.size(1)).bool().to(speech_features.device)
+            return speech_features, speech_attn_mask
+        else:
+            speech_features = []
+            speech_attn_mask = []
+            for wav in wav_list:
+                speech_feature = self.process_audio_for_llm_input(wav)
+                speech_features.append(speech_feature)
+                speech_attn_mask.append(torch.zeros(1,speech_feature.size(1)).bool())
+
+            speech_features = self.pad_sequence(speech_features,padding_side='right',padding_value=0.0)
+            speech_attn_mask = self.pad_sequence(speech_attn_mask,padding_side='right',padding_value=True).squeeze(1)
+            return speech_features, speech_attn_mask
+        
+    def process_audio_for_llm_input(self, wav: torch.Tensor):
+        n_frames = 3000
+        min_length = 16000
+        wav = wav.flatten()
+
+        if wav.shape[0] < min_length:
+            wav = F.pad(wav, (0, min_length - wav.shape[0]))
+        
+        mels = whisper.log_mel_spectrogram(wav, n_mels=self.n_mels).unsqueeze(0).to(self._device)
+        if mels.shape[-1] > n_frames:
+            mel_segments = []
+            # Segment and process long audio
+            for i in range(0, mels.shape[-1], n_frames):
+                mel = mels[:,:,i:i+n_frames]
+                if mel.shape[-1] < n_frames:
+                    mel = self.pad_or_trim(mel,n_frames)
+                mel_segments.append(mel)
+
+            if self.low_resource:
+                audio_features = [self._process_mel_segment(mel) for mel in mel_segments]
+                speech_tokens = torch.cat(audio_features, dim=1)
+            else:
+                # Batch Inference Mode
+                mel_segments = torch.cat(mel_segments,dim=0)
+                B, _, _ = mel_segments.shape
+                audio_features = self._process_mel_segment(mel_segments)
+                speech_tokens = audio_features.view(1, B * self.compression_size, self.n_state)
+        else:
+            if mels.shape[-1] < n_frames:
+                mels = self.pad_or_trim(mels,n_frames)
+            speech_tokens = self._process_mel_segment(mels)
+            
+        return self.llm_proj(speech_tokens)
+
+    def _process_mel_segment(self, mel_segment: torch.Tensor):        
+        # Feature extraction and hierarchical compression
+        audio_feature = self.embed_audio(mel_segment)
+        
+        stage_1_token = self.compressor1(x=audio_feature)
+        stage_1_feature = self.stage1(audio_feature.transpose(1, 2))
+        stage_2_token = self.compressor2(x=stage_1_feature)
+        stage_2_feature = self.stage2(stage_1_feature.transpose(1, 2))
+        stage_3_token = self.compressor3(x=stage_2_feature)
+
+        stage_tokens = torch.cat([stage_1_token, stage_2_token, stage_3_token], dim=1)
+        compressed_tokens = self.compressor(stage_tokens)
+
+        # Cross-attention with hierarchical features
+        h_audio_feature = torch.cat([audio_feature, stage_1_feature, stage_2_feature], dim=1)
+        for block in self.out_attn:
+            compressed_tokens = block(x=compressed_tokens, xa=h_audio_feature)
+        
+        return compressed_tokens
+        
+    def pad_sequence(self, sequences, padding_side='right', padding_value=0.0):
+        max_len = max(seq.size(1) for seq in sequences)
+        output_dims = (len(sequences), max_len) + sequences[0].shape[2:]
+        output = torch.full(output_dims, padding_value, dtype=sequences[0].dtype, device=sequences[0].device)
+        
+        for i, seq in enumerate(sequences):
+            length = seq.size(1)
+            if padding_side == 'right':
+                output[i, :length, ...] = seq
+            else:
+                output[i, -length:, ...] = seq
+        return output
+    
+    def pad_or_trim(self, array, length: int = 480000, *, axis: int = -1):
+        """
+        Pad or trim the audio array to N_SAMPLES, as expected by the encoder.
+        """
+        if torch.is_tensor(array):
+            pad_widths = [(0, 0)] * array.ndim
+            pad_widths[axis] = (0, length - array.shape[axis])
+            array = F.pad(array, [pad for sizes in pad_widths[::-1] for pad in sizes])
+        else:
+            pad_widths = [(0, 0)] * array.ndim
+            pad_widths[axis] = (0, length - array.shape[axis])
+            array = np.pad(array, pad_widths)
+        return array
+# --- Main Model Class ---
+
+class FastSLMPreTrainedModel(PreTrainedModel):
+    config_class = FastSLMConfig
+    base_model_prefix = "fastslm"
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            nn.init.normal_(module.weight, std=0.02)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+
+class FastSLMForConditionalGeneration(FastSLMPreTrainedModel, GenerationMixin):
+    config_class = FastSLMConfig 
+    def __init__(self, config: FastSLMConfig):
+        super().__init__(config)
+        
+        # Initialize the two main components using their respective sub-configs
+        self.encoder = SpeechEncoder(config)
+        self.llm = AutoModelForCausalLM.from_config(
+            config.llm_config,
+            trust_remote_code=True
+        )
+        if self.llm._tied_weights_keys is not None:
+            self._tied_weights_keys = [f"llm.{k}" for k in self.llm._tied_weights_keys]
+
+        llm_lora_config = LoraConfig(
+        r=config.lora_r,           
+        lora_alpha=config.lora_a,
+        target_modules=config.llm_modules,
+        lora_dropout=0.01,  
+        task_type="CAUSAL_LM",
+        )
+        self.llm = get_peft_model(self.llm, llm_lora_config)
+
+        self.tokenizer = AutoTokenizer.from_pretrained(config.llm_config._name_or_path, use_fast=False, trust_remote_code=True)
+        # Add special tokens
+        audio_token = ['<|AUDIO|>', '<|audio_bos|>', '<|audio_eos|>']
+        task_token = ['<|ASR|>', '<|AST|>', '<|SSUM|>', '<|SQQA|>']
+        language_token = [f"<|{lang.upper()}|>" for lang in LANGUAGES]
+        special_tokens = audio_token + language_token + task_token
+        self.tokenizer.add_special_tokens({"additional_special_tokens": special_tokens})
+
+    def get_input_embeddings(self) -> nn.Module:
+        """Returns the input embedding layer of the LLM."""
+        return self.llm.get_input_embeddings()
+
+    def set_input_embeddings(self, value: nn.Module):
+        """Sets the input embedding layer of the LLM."""
+        self.llm.set_input_embeddings(value)
+
+    def process_audio(self, audio_array: np.ndarray, sample_rate: int) -> torch.Tensor:
+        audio = torch.tensor(audio_array, dtype=torch.float32)
+        if sample_rate != 16000:
+            resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=16000)
+            audio = resampler(audio)
+        return audio
+    
+    def save_pretrained(self, save_directory, **kwargs):
+        super().save_pretrained(save_directory, **kwargs)
+        if hasattr(self.llm, "save_pretrained"):
+            self.llm.save_pretrained(f"{save_directory}/llm")
+
+    def forward(
+        self,
+        audio: List[torch.Tensor],
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        **kwargs
+    ):
+        speech_query, speech_attn_mask = self.encoder(audio)
+
+        token_embedding = self.llm.get_input_embeddings()
+        
+        # Create speech labels (-100 to ignore in loss calculation)
+        speech_label_len = int(speech_query.shape[1])
+        speech_labels = torch.full(
+            (speech_query.shape[0], speech_label_len),
+            fill_value=-100,
+            dtype=torch.long,
+            device=speech_query.device
+        )
+
+        audio_token_id = self.tokenizer.convert_tokens_to_ids("<|AUDIO|>")
+        idx = torch.nonzero(input_ids[0] == audio_token_id)[0][0].item()
+        left_token, right_token = input_ids[:,:idx], input_ids[:,idx+1:]
+
+        left_label, right_label = labels[:,:idx], labels[:,idx+1:]
+        left_embed = token_embedding(left_token.long()).to(speech_query.device)
+        right_embed = token_embedding(right_token.long()).to(speech_query.device)
+
+        left_mask = (left_token != self.tokenizer.pad_token_id).long().to(self.device)
+        right_mask = (right_token != self.tokenizer.pad_token_id).long().to(self.device)
+        speech_attn_mask = (speech_attn_mask.int() <= 0).long()
+
+        inputs_embeds = torch.cat([left_embed,speech_query,right_embed],dim=1)
+        labels = torch.cat([left_label,speech_labels,right_label], dim=1).long()
+        attention_mask = torch.cat([
+            left_mask, speech_attn_mask, right_mask
+            ], dim=1
+        )
+
+        outputs = self.llm(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            labels=labels,
+            return_dict=True,
+        )
+        return outputs
+
+    def generate(self, input_ids, audio: List[torch.Tensor] = None, **kwargs):
+        token_embedding = self.llm.get_input_embeddings()
+        if audio is not None:
+            speech_query, speech_attn_mask = self.encoder(audio)
+            audio_token_id = self.tokenizer.convert_tokens_to_ids("<|AUDIO|>")
+            idx = torch.nonzero(input_ids[0] == audio_token_id)[0][0].item()
+
+            left_embed = token_embedding(input_ids[:, :idx])
+            right_embed = token_embedding(input_ids[:, idx+1:])
+
+            input_embeds = torch.cat([left_embed, speech_query, right_embed], dim=1)
+            
+            # Create attention mask
+            left_mask = torch.ones_like(input_ids[:, :idx]).to(input_ids.device)
+            right_mask = torch.ones_like(input_ids[:, idx+1:]).to(input_ids.device)
+            attention_mask = torch.cat([left_mask, (~speech_attn_mask).long().to(input_ids.device), right_mask], dim=1)
+
+            generated_ids = self.llm.generate(
+                inputs_embeds=input_embeds,
+                attention_mask=attention_mask,
+                pad_token_id=self.tokenizer.eos_token_id,
+                **kwargs
+            )
+        else:
+            input_embeds = token_embedding(input_ids)
+            attention_mask = torch.ones([
+                input_embeds.size(0), input_embeds.size(1)], dtype=torch.long, device=input_embeds.device
+            )
+            with self.llm.disable_adapter():
+                generated_ids = self.llm.generate(
+                    inputs_embeds=input_embeds,
+                    attention_mask=attention_mask,
+                    pad_token_id=self.tokenizer.eos_token_id,
+                    **kwargs
+                )
+        return generated_ids
+
+    def pad_embeddings(self, sequences, padding_side='right', padding_value=0.0):
+        """Pads a list of tensors to the same length."""
+        max_len = max(seq.size(0) for seq in sequences)
+        output_dims = (len(sequences), max_len) + sequences[0].shape[1:]
+        output = torch.full(output_dims, padding_value, dtype=sequences[0].dtype, device=sequences[0].device)
+        
+        for i, seq in enumerate(sequences):
+            length = seq.size(0)
+            if padding_side == 'right':
+                output[i, :length, ...] = seq
+            else:
+                output[i, -length:, ...] = seq
+        return output
+
+# Register the model with AutoModelForCausalLM
+AutoConfig.register("fastslm", FastSLMConfig)
+AutoModelForCausalLM.register(FastSLMConfig, FastSLMForConditionalGeneration)

modeling_whisper.py

@@ -0,0 +1,200 @@
+import base64
+import gzip
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Dict, Iterable, Optional, Tuple
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import Tensor, nn
+
+try:
+    from torch.nn.functional import scaled_dot_product_attention
+
+    SDPA_AVAILABLE = True
+except (ImportError, RuntimeError, OSError):
+    scaled_dot_product_attention = None
+    SDPA_AVAILABLE = False
+
+
+@dataclass
+class ModelDimensions:
+    n_mels: int
+    n_audio_ctx: int
+    n_audio_state: int
+    n_audio_head: int
+    n_audio_layer: int
+    n_vocab: int
+    n_text_ctx: int
+    n_text_state: int
+    n_text_head: int
+    n_text_layer: int
+
+
+class LayerNorm(nn.LayerNorm):
+    def forward(self, x: Tensor) -> Tensor:
+        return super().forward(x.float()).type(x.dtype)
+
+
+class Linear(nn.Linear):
+    def forward(self, x: Tensor) -> Tensor:
+        return F.linear(
+            x,
+            self.weight.to(x.dtype),
+            None if self.bias is None else self.bias.to(x.dtype),
+        )
+
+
+class Conv1d(nn.Conv1d):
+    def _conv_forward(
+        self, x: Tensor, weight: Tensor, bias: Optional[Tensor]
+    ) -> Tensor:
+        return super()._conv_forward(
+            x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)
+        )
+
+
+def sinusoids(length, channels, max_timescale=10000):
+    """Returns sinusoids for positional embedding"""
+    assert channels % 2 == 0
+    log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)
+    inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))
+    scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]
+    return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)
+
+
+@contextmanager
+def disable_sdpa():
+    prev_state = MultiHeadAttention.use_sdpa
+    try:
+        MultiHeadAttention.use_sdpa = False
+        yield
+    finally:
+        MultiHeadAttention.use_sdpa = prev_state
+
+
+class MultiHeadAttention(nn.Module):
+    use_sdpa = True
+
+    def __init__(self, n_state: int, n_head: int):
+        super().__init__()
+        self.n_head = n_head
+        self.query = Linear(n_state, n_state)
+        self.key = Linear(n_state, n_state, bias=False)
+        self.value = Linear(n_state, n_state)
+        self.out = Linear(n_state, n_state)
+
+    def forward(
+        self,
+        x: Tensor,
+        xa: Optional[Tensor] = None,
+        mask: Optional[Tensor] = None,
+        kv_cache: Optional[dict] = None,
+    ):
+        q = self.query(x)
+
+        if kv_cache is None or xa is None or self.key not in kv_cache:
+            # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;
+            # otherwise, perform key/value projections for self- or cross-attention as usual.
+            k = self.key(x if xa is None else xa)
+            v = self.value(x if xa is None else xa)
+        else:
+            # for cross-attention, calculate keys and values once and reuse in subsequent calls.
+            k = kv_cache[self.key]
+            v = kv_cache[self.value]
+
+        wv, qk = self.qkv_attention(q, k, v, mask)
+        return self.out(wv), qk
+
+    def qkv_attention(
+        self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        n_batch, n_ctx, n_state = q.shape
+        scale = (n_state // self.n_head) ** -0.25
+        q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
+        k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
+        v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
+
+        if SDPA_AVAILABLE and MultiHeadAttention.use_sdpa:
+            a = scaled_dot_product_attention(
+                q, k, v, is_causal=mask is not None and n_ctx > 1
+            )
+            out = a.permute(0, 2, 1, 3).flatten(start_dim=2)
+            qk = None
+        else:
+            qk = (q * scale) @ (k * scale).transpose(-1, -2)
+            if mask is not None:
+                qk = qk + mask[:n_ctx, :n_ctx]
+            qk = qk.float()
+
+            w = F.softmax(qk, dim=-1).to(q.dtype)
+            out = (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2)
+            qk = qk.detach()
+
+        return out, qk
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, n_state: int, n_head: int, cross_attention: bool = False):
+        super().__init__()
+
+        self.attn = MultiHeadAttention(n_state, n_head)
+        self.attn_ln = LayerNorm(n_state)
+
+        self.cross_attn = (
+            MultiHeadAttention(n_state, n_head) if cross_attention else None
+        )
+        self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None
+
+        n_mlp = n_state * 4
+        self.mlp = nn.Sequential(
+            Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state)
+        )
+        self.mlp_ln = LayerNorm(n_state)
+
+    def forward(
+        self,
+        x: Tensor,
+        xa: Optional[Tensor] = None,
+        mask: Optional[Tensor] = None,
+        kv_cache: Optional[dict] = None,
+    ):
+        x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0]
+        if self.cross_attn:
+            x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0]
+        x = x + self.mlp(self.mlp_ln(x))
+        return x
+
+
+class AudioEncoder(nn.Module):
+    def __init__(
+        self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int
+    ):
+        super().__init__()
+        self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
+        self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
+        self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state))
+
+        self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(
+            [ResidualAttentionBlock(n_state, n_head) for _ in range(n_layer)]
+        )
+        self.ln_post = LayerNorm(n_state)
+
+    def forward(self, x: Tensor):
+        """
+        x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
+            the mel spectrogram of the audio
+        """
+        x = F.gelu(self.conv1(x))
+        x = F.gelu(self.conv2(x))
+        x = x.permute(0, 2, 1)
+
+        assert x.shape[1:] == self.positional_embedding.shape, "incorrect audio shape"
+        x = (x + self.positional_embedding).to(x.dtype)
+
+        for block in self.blocks:
+            x = block(x)
+
+        x = self.ln_post(x)
+        return x

special_tokens_map.json

@@ -0,0 +1,81 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|AUDIO|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|audio_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|audio_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|EN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|KO|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|ASR|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|AST|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|SSUM|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|SQQA|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,307 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<|AUDIO|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|audio_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|audio_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|EN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|KO|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|ASR|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|AST|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<|SSUM|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<|SQQA|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|AUDIO|>",
+    "<|audio_bos|>",
+    "<|audio_eos|>",
+    "<|EN|>",
+    "<|KO|>",
+    "<|ASR|>",
+    "<|AST|>",
+    "<|SSUM|>",
+    "<|SQQA|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}