Liquid-Codec / configuration_lfm3.py
eustlb's picture
eustlb HF Staff
Mirror LiquidAI/Liquid-Codec @ b5abfac4
8ce71a5 verified
Raw History Blame Contribute Delete
4.37 kB
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# This file was automatically generated from src/transformers/models/lfm3/modular_lfm3.py.
# Do NOT edit this file manually as any edits will be overwritten by the generation of
# the file from the modular. If any change should be done, please apply the change to the
# modular_lfm3.py file directly. One of our CI enforces this.
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2026 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from huggingface_hub.dataclasses import strict
from transformers.configuration_utils import PreTrainedConfig
from transformers.utils import auto_docstring
@auto_docstring(checkpoint="LiquidAI/LFM3-1.7B")
@strict
class Lfm3Config(PreTrainedConfig):
r"""
conv_bias (`bool`, *optional*, defaults to `False`):
Whether to use bias in the conv layers.
conv_L_cache (`int`, *optional*, defaults to 7):
L_cache dim in the conv layers.
use_sandwich_norm (`bool`, *optional*, defaults to `True`):
Whether to apply an RMSNorm to the operator and feed-forward outputs before they enter the residual stream.
use_attention_gate (`bool`, *optional*, defaults to `True`):
Whether the attention layers scale each head's output by a sigmoid gate computed from the layer input.
```python
>>> from transformers import Lfm3Model, Lfm3Config
>>> # Initializing a LFM3 model
>>> configuration = Lfm3Config()
>>> # Initializing a model from the LFM3-1.7B style configuration
>>> model = Lfm3Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```
"""
model_type = "lfm3"
keys_to_ignore_at_inference = ["past_key_values"]
default_theta = 1000000.0
vocab_size: int = 128_000
hidden_size: int = 2048
intermediate_size: int = 7680
num_hidden_layers: int = 24
num_attention_heads: int = 16
num_key_value_heads: int = 4
max_position_embeddings: int = 128_000
initializer_range: float = 0.02
norm_eps: float = 0.00001
use_cache: bool = True
pad_token_id: int | None = 124_893
bos_token_id: int | None = 124_894
eos_token_id: int | list[int] | None = 124_900
tie_word_embeddings: bool = True
rope_parameters: dict | None = None
conv_bias: bool = False
conv_L_cache: int = 7
layer_types: list[str] | None = None
attribute_map = {"rms_norm_eps": "norm_eps"}
_rope_type_labels = ("full_attention", "sliding_attention")
head_dim: int = 128
sliding_window: int | None = 256
use_sandwich_norm: bool = True
use_attention_gate: bool = True
attention_bias: bool = False
attention_dropout: float = 0.0
def __post_init__(self, **kwargs):
if self.rope_parameters is None:
self.rope_parameters = {
"sliding_attention": {"rope_type": "default", "rope_theta": 10_000.0},
"full_attention": {
"rope_type": "proportional",
"rope_theta": 1_000_000.0,
"partial_rotary_factor": 0.5,
},
}
if self.layer_types is None:
self.layer_types = [
"full_attention" if i % 4 == 2 else "sliding_attention" if i % 4 == 0 and i > 0 else "conv"
for i in range(self.num_hidden_layers)
]
super().__post_init__(**kwargs)
__all__ = ["Lfm3Config"]