Spaces:
Running on Zero
Running on Zero
Download models/breeze.py from Quantumbraid/agent.voice: direct link, hf CLI and curl.
- Browser
- Download file 81.8 kB
-
https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/models/breeze.py
- Command line
-
hf download hf://spaces/Quantumbraid/agent.voice/models/breeze.py
-
curl -L -o breeze.py https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/models/breeze.py
81.8 kB
# Breeze model implementation adapted from Hugging Face Transformers.
#
# coding=utf-8
# Copyright 2025 Sesame and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
import torch
from torch import nn
from transformers.activations import ACT2FN
from transformers.cache_utils import Cache, DynamicCache
from transformers.generation import GenerationMixin
from transformers.integrations import use_kernel_forward_from_hub
from transformers.masking_utils import create_causal_mask
from transformers.modeling_layers import GradientCheckpointingLayer
from transformers.modeling_outputs import (
BaseModelOutputWithPast,
CausalLMOutputWithPast,
)
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
from transformers.models.auto import AutoModel
from transformers.processing_utils import Unpack
from transformers.utils import (
ModelOutput,
TransformersKwargs,
auto_docstring,
can_return_tuple,
logging,
)
from transformers.utils.deprecation import deprecate_kwarg
from .breeze_config import BreezeConfig, BreezeDepthDecoderConfig
from .generation_breeze import BreezeGenerationMixin
logger = logging.get_logger(__name__)
@dataclass
@auto_docstring(
custom_intro="""
Base class for the model autoregressive outputs.
"""
)
class BreezeOutputWithPast(ModelOutput):
r"""
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
`(batch_size, num_heads, sequence_length, embed_size_per_head)`)
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
depth_decoder_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
Language modeling loss (for next-token prediction) of the depth decoder model.
depth_decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the depth decoder (scores for each vocabulary token before SoftMax).
depth_decoder_past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
`(batch_size, num_heads, sequence_length, embed_size_per_head)`)
depth_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
depth_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
sequence_length)`.
backbone_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
Language modeling loss (for next-token prediction) of the backbone model.
"""
loss: torch.FloatTensor | None = None
logits: torch.FloatTensor = None
past_key_values: tuple[tuple[torch.FloatTensor]] | None = None
hidden_states: tuple[torch.FloatTensor, ...] | None = None
attentions: tuple[torch.FloatTensor, ...] | None = None
depth_decoder_loss: torch.FloatTensor | None = None
depth_decoder_logits: torch.FloatTensor = None
depth_decoder_past_key_values: tuple[tuple[torch.FloatTensor]] | None = None
depth_decoder_hidden_states: tuple[torch.FloatTensor, ...] | None = None
depth_decoder_attentions: tuple[torch.FloatTensor, ...] | None = None
backbone_loss: torch.FloatTensor | None = None
@use_kernel_forward_from_hub("RMSNorm")
class BreezeRMSNorm(nn.Module):
def __init__(self, hidden_size, eps=1e-6):
"""
BreezeRMSNorm is equivalent to T5LayerNorm
"""
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states):
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
def extra_repr(self):
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
class BreezeRotaryEmbedding(nn.Module):
inv_freq: torch.Tensor # fix linting for `register_buffer`
def __init__(self, config: BreezeConfig, device=None):
super().__init__()
# BC: "rope_type" was originally "type"
if hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
self.rope_type = config.rope_scaling.get(
"rope_type", config.rope_scaling.get("type")
)
else:
self.rope_type = "default"
self.max_seq_len_cached = config.max_position_embeddings
self.original_max_seq_len = config.max_position_embeddings
self.config = config
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
self.register_buffer("inv_freq", inv_freq, persistent=False)
self.original_inv_freq = self.inv_freq
@torch.no_grad()
@dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
def forward(self, x, position_ids):
inv_freq_expanded = (
self.inv_freq[None, :, None]
.float()
.expand(position_ids.shape[0], -1, 1)
.to(x.device)
)
position_ids_expanded = position_ids[:, None, :].float()
device_type = (
x.device.type
if isinstance(x.device.type, str) and x.device.type != "mps"
else "cpu"
)
with torch.autocast(device_type=device_type, enabled=False): # Force float32
freqs = (
inv_freq_expanded.float() @ position_ids_expanded.float()
).transpose(1, 2)
emb = torch.cat((freqs, freqs), dim=-1)
cos = emb.cos() * self.attention_scaling
sin = emb.sin() * self.attention_scaling
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
class BreezeMLP(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
self.intermediate_size = config.intermediate_size
self.gate_proj = nn.Linear(
self.hidden_size, self.intermediate_size, bias=config.mlp_bias
)
self.up_proj = nn.Linear(
self.hidden_size, self.intermediate_size, bias=config.mlp_bias
)
self.down_proj = nn.Linear(
self.intermediate_size, self.hidden_size, bias=config.mlp_bias
)
self.act_fn = ACT2FN[config.hidden_act]
def forward(self, x):
down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
return down_proj
def rotate_half(x):
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
"""Applies Rotary Position Embedding to the query and key tensors.
Args:
q (`torch.Tensor`): The query tensor.
k (`torch.Tensor`): The key tensor.
cos (`torch.Tensor`): The cosine part of the rotary embedding.
sin (`torch.Tensor`): The sine part of the rotary embedding.
position_ids (`torch.Tensor`, *optional*):
Deprecated and unused.
unsqueeze_dim (`int`, *optional*, defaults to 1):
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
"""
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)
q_embed = (q * cos) + (rotate_half(q) * sin)
k_embed = (k * cos) + (rotate_half(k) * sin)
return q_embed, k_embed
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
"""
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
"""
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
if n_rep == 1:
return hidden_states
hidden_states = hidden_states[:, :, None, :, :].expand(
batch, num_key_value_heads, n_rep, slen, head_dim
)
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
def eager_attention_forward(
module: nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor | None,
scaling: float,
dropout: float = 0.0,
**kwargs: Unpack[TransformersKwargs],
):
key_states = repeat_kv(key, module.num_key_value_groups)
value_states = repeat_kv(value, module.num_key_value_groups)
attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
if attention_mask is not None:
causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
attn_weights = attn_weights + causal_mask
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
query.dtype
)
attn_weights = nn.functional.dropout(
attn_weights, p=dropout, training=module.training
)
attn_output = torch.matmul(attn_weights, value_states)
attn_output = attn_output.transpose(1, 2).contiguous()
return attn_output, attn_weights
class BreezeAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
def __init__(self, config: BreezeConfig, layer_idx: int):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.head_dim = getattr(
config, "head_dim", config.hidden_size // config.num_attention_heads
)
self.num_key_value_groups = (
config.num_attention_heads // config.num_key_value_heads
)
self.scaling = self.head_dim**-0.5
self.attention_dropout = config.attention_dropout
self.is_causal = True
self.q_proj = nn.Linear(
config.hidden_size,
config.num_attention_heads * self.head_dim,
bias=config.attention_bias,
)
self.k_proj = nn.Linear(
config.hidden_size,
config.num_key_value_heads * self.head_dim,
bias=config.attention_bias,
)
self.v_proj = nn.Linear(
config.hidden_size,
config.num_key_value_heads * self.head_dim,
bias=config.attention_bias,
)
self.o_proj = nn.Linear(
config.num_attention_heads * self.head_dim,
config.hidden_size,
bias=config.attention_bias,
)
@deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
attention_mask: torch.Tensor | None,
past_key_values: Cache | None = None,
cache_position: torch.LongTensor | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> tuple[torch.Tensor, torch.Tensor]:
input_shape = hidden_states.shape[:-1]
hidden_shape = (*input_shape, -1, self.head_dim)
query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
cos, sin = position_embeddings
query_states, key_states = apply_rotary_pos_emb(
query_states, key_states, cos, sin
)
if past_key_values is not None:
# sin and cos are specific to RoPE models; cache_position needed for the static cache
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
key_states, value_states = past_key_values.update(
key_states, value_states, self.layer_idx, cache_kwargs
)
attention_interface: Callable = eager_attention_forward
if self.config._attn_implementation != "eager":
attention_interface = ALL_ATTENTION_FUNCTIONS[
self.config._attn_implementation
]
attn_output, attn_weights = attention_interface(
self,
query_states,
key_states,
value_states,
attention_mask,
dropout=0.0 if not self.training else self.attention_dropout,
scaling=self.scaling,
**kwargs,
)
attn_output = attn_output.reshape(*input_shape, -1).contiguous()
attn_output = self.o_proj(attn_output)
return attn_output, attn_weights
class BreezeDecoderLayer(GradientCheckpointingLayer):
def __init__(self, config: BreezeConfig, layer_idx: int):
super().__init__()
self.hidden_size = config.hidden_size
self.self_attn = BreezeAttention(config=config, layer_idx=layer_idx)
self.mlp = BreezeMLP(config)
self.input_layernorm = BreezeRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.post_attention_layernorm = BreezeRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
@deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
past_key_values: Cache | None = None,
use_cache: bool | None = False,
cache_position: torch.LongTensor | None = None,
position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, # necessary, but kept here for BC
**kwargs: Unpack[TransformersKwargs],
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
# Self Attention
hidden_states, _ = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=past_key_values,
use_cache=use_cache,
cache_position=cache_position,
position_embeddings=position_embeddings,
**kwargs,
)
hidden_states = residual + hidden_states
# Fully Connected
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
@auto_docstring(
custom_intro="""
The bare Breeze Model outputting raw hidden-states without any specific head on top.
"""
)
@auto_docstring
class BreezePreTrainedModel(PreTrainedModel):
config_class = BreezeConfig
config: BreezeConfig
base_model_prefix = "model"
supports_gradient_checkpointing = True
_no_split_modules = ["BreezeDecoderLayer"]
_skip_keys_device_placement = ["past_key_values"]
_supports_flash_attn = True
_supports_sdpa = True
# does not because of Mimi codec model
# _supports_flex_attn = True
_can_compile_fullgraph = True
_supports_attention_backend = True
_can_record_outputs = {
"hidden_states": BreezeDecoderLayer,
"attentions": BreezeAttention,
}
def _init_weights(self, module):
super()._init_weights(module)
if isinstance(module, BreezeCodebooksHead):
num_codebooks = module.num_codebooks
for i in range(num_codebooks - 1):
module.weight.data[i].normal_(
mean=0.0, std=self.config.initializer_range
)
@auto_docstring
class BreezeDepthDecoderModel(BreezePreTrainedModel):
config: BreezeDepthDecoderConfig
def __init__(self, config):
super().__init__(config)
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
if hasattr(config, "audio_embed_size") and config.audio_embed_size:
self.audio_embed_size = config.audio_embed_size
else:
self.audio_embed_size = config.backbone_hidden_size
self.embed_tokens = nn.Embedding(
(config.num_codebooks * config.vocab_size), self.audio_embed_size
)
self.layers = nn.ModuleList(
[
BreezeDecoderLayer(config, layer_idx)
for layer_idx in range(config.num_hidden_layers)
]
)
self.norm = BreezeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = BreezeRotaryEmbedding(config=config)
self.gradient_checkpointing = False
self.inputs_embeds_projector = nn.Linear(
self.audio_embed_size, config.hidden_size, bias=False
)
if config.backbone_hidden_size != self.audio_embed_size:
self.backbone_hidden_state_projector = nn.Linear(
config.backbone_hidden_size, self.audio_embed_size, bias=False
)
else:
self.backbone_hidden_state_projector = None
# Initialize weights and apply final processing
self.post_init()
@auto_docstring
def forward(
self,
input_ids: torch.LongTensor = None,
backbone_last_hidden_state: torch.FloatTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
past_key_values: Cache | None = None,
inputs_embeds: torch.FloatTensor | None = None,
use_cache: bool | None = None,
cache_position: torch.LongTensor | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> tuple | BaseModelOutputWithPast:
r"""
backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*):
The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model)
is provided in the `input_ids` argument.
"""
if position_ids is not None and not torch.compiler.is_compiling():
logger.warning_once(
"Custom `position_ids` were provided but will be ignored. Breeze depth decoder automatically determines position_ids "
"from `cache_position` and as it requires them to be identical across the batch, the provided position_ids will be ignored."
)
position_ids = None
if (input_ids is None) ^ (inputs_embeds is not None):
raise ValueError(
"You must specify exactly one of input_ids or inputs_embeds."
)
if use_cache and past_key_values is None:
past_key_values = DynamicCache(config=self.config)
if cache_position is None:
past_seen_tokens = (
past_key_values.get_seq_length() if past_key_values is not None else 0
)
inputs_seq_length = (
inputs_embeds.shape[1]
if inputs_embeds is not None
else input_ids.shape[1]
)
device = (
inputs_embeds.device if inputs_embeds is not None else input_ids.device
)
cache_position = torch.arange(
past_seen_tokens, past_seen_tokens + inputs_seq_length, device=device
)
if inputs_embeds is None:
codebook_idxs = torch.clamp(cache_position - 1, min=0)
offset = codebook_idxs * self.vocab_size
inputs_embeds = self.embed_tokens(input_ids + offset)
input_ids_are_first_codebook = cache_position[0] == 0
if backbone_last_hidden_state is not None:
if self.backbone_hidden_state_projector is not None:
inputs_embeds[:, 0] = self.backbone_hidden_state_projector(
backbone_last_hidden_state
)
else:
inputs_embeds[:, 0] = backbone_last_hidden_state
else:
if not torch.compiler.is_compiling() and input_ids_are_first_codebook:
logger.warning(
"When the first codebook token is provided, `backbone_last_hidden_state` should also be provided for correct inference."
)
inputs_embeds = self.inputs_embeds_projector(inputs_embeds)
causal_mask = create_causal_mask(
config=self.config,
input_embeds=inputs_embeds,
attention_mask=attention_mask,
cache_position=cache_position,
past_key_values=past_key_values,
position_ids=position_ids,
)
hidden_states = inputs_embeds
# create position embeddings to be shared across the decoder layers
position_ids = cache_position.unsqueeze(0)
position_embeddings = self.rotary_emb(hidden_states, position_ids)
for decoder_layer in self.layers[: self.config.num_hidden_layers]:
hidden_states = decoder_layer(
hidden_states,
attention_mask=causal_mask,
position_ids=position_ids,
past_key_values=past_key_values,
use_cache=use_cache,
cache_position=cache_position,
position_embeddings=position_embeddings,
**kwargs,
)
hidden_states = self.norm(hidden_states)
return BaseModelOutputWithPast(
last_hidden_state=hidden_states,
past_key_values=past_key_values if use_cache else None,
)
class BreezeCodebooksHead(nn.Module):
def __init__(self, hidden_size, num_codebooks, vocab_size):
super().__init__()
self.num_codebooks = num_codebooks
self.weight = nn.Parameter(
torch.empty(self.num_codebooks - 1, hidden_size, vocab_size)
)
def forward(self, hidden_states, cache_position=None):
if cache_position is None:
seq_length = hidden_states.shape[1]
codebook_weight = self.weight[torch.arange(seq_length)]
else:
codebook_idxs = cache_position - 1
codebook_weight = self.weight[codebook_idxs]
hidden_states = [
nn.functional.linear(
hidden_states[:, codebook_idx, :], codebook_weight[codebook_idx].T
)
for codebook_idx in range(codebook_weight.shape[0])
]
hidden_states = torch.stack(hidden_states, dim=1)
return hidden_states
# 重写 extra_repr 方法以显示参数细节
def extra_repr(self):
return f"weight_shape={list(self.weight.shape)}, num_codebooks={self.num_codebooks}"
@auto_docstring(
custom_intro="""
The BreezeDepthDecoder Model transformer, with a [`BreezeCodebooksHead`] on top,
which can be seen a position-specific language modeling head, allowing to use a different linear layer for each codebook
(e.g. position 0 is the first codebook and uses the first codebook head, etc.)
"""
)
class BreezeDepthDecoderForCausalLM(BreezePreTrainedModel, GenerationMixin):
_tied_weights_keys = None
_tp_plan = None
_pp_plan = None
def __init__(self, config):
super().__init__(config)
self.model = BreezeDepthDecoderModel(config)
self.vocab_size = config.vocab_size
self.codebooks_head = BreezeCodebooksHead(
config.hidden_size, config.num_codebooks, config.vocab_size
)
# Compute per-codebook loss weights
# The depth decoder predicts num_codebooks-1 codebooks (31 for default config)
# We distribute weights by prepending a virtual codebook to make division even
codebook_loss_weights_list = getattr(config, "codebook_loss_weights", [1.0])
num_depth_codebooks = config.num_codebooks - 1 # 31 for default 32 codebooks
num_weights = len(codebook_loss_weights_list)
num_codebooks_with_virtual = num_depth_codebooks + 1
codebooks_per_weight = num_codebooks_with_virtual // num_weights
weights_expanded = []
for weight in codebook_loss_weights_list:
weights_expanded.extend([weight] * codebooks_per_weight)
weights_expanded = weights_expanded[1:]
self.codebook_loss_weights = weights_expanded
# Initialize weights and apply final processing
self.post_init()
@can_return_tuple
@auto_docstring
def forward(
self,
input_ids: torch.LongTensor = None,
backbone_last_hidden_state: torch.FloatTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
past_key_values: Cache | list[torch.FloatTensor] | None = None,
inputs_embeds: torch.FloatTensor | None = None,
labels: torch.LongTensor | None = None,
use_cache: bool | None = None,
cache_position: torch.LongTensor | None = None,
logits_to_keep: int | torch.Tensor = 0,
**kwargs: Unpack[TransformersKwargs],
) -> tuple | CausalLMOutputWithPast:
r"""
backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*):
The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model)
is provided in the `input_ids` argument.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
"""
outputs = self.model(
input_ids=input_ids,
backbone_last_hidden_state=backbone_last_hidden_state,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=past_key_values,
inputs_embeds=inputs_embeds,
use_cache=use_cache,
cache_position=cache_position,
**kwargs,
)
hidden_states = outputs[0]
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
if isinstance(logits_to_keep, int):
if logits_to_keep == 0:
# skip idx 0 logits since it's for the concatenated backbone last hidden state
slice_indices = slice(1, None)
else:
slice_indices = slice(-logits_to_keep, None)
else:
slice_indices = logits_to_keep
loss = None
logits = None
logits = self.codebooks_head(
hidden_states[:, slice_indices, :],
cache_position[slice_indices] if cache_position is not None else None,
)
logits = logits.contiguous()
if labels is not None:
shift_labels = labels[..., 1:].contiguous()
loss = self.loss_function(
logits=logits,
labels=None,
vocab_size=self.config.vocab_size,
shift_labels=shift_labels,
**kwargs,
)
return CausalLMOutputWithPast(
loss=loss,
logits=logits,
past_key_values=outputs.past_key_values,
hidden_states=outputs.hidden_states,
attentions=outputs.attentions,
)
def prepare_inputs_for_generation(
self,
input_ids: torch.LongTensor,
past_key_values: Cache | None = None,
attention_mask: torch.LongTensor | None = None,
inputs_embeds: torch.FloatTensor | None = None,
cache_position: torch.LongTensor | None = None,
**kwargs,
):
model_inputs = super().prepare_inputs_for_generation(
input_ids,
past_key_values,
attention_mask,
inputs_embeds,
cache_position,
**kwargs,
)
is_first_generation_step = model_inputs["cache_position"][0] == 0
if not is_first_generation_step:
model_inputs.pop("backbone_last_hidden_state")
# breeze depth decoder does not use position_ids
model_inputs.pop("position_ids")
return model_inputs
class BreezeBackboneModelEmbeddings(nn.Module):
def __init__(self, config):
super().__init__()
self.hidden_size = config.hidden_size
if hasattr(config, "audio_embed_size") and config.audio_embed_size:
self.audio_embed_size = config.audio_embed_size
else:
self.audio_embed_size = self.hidden_size
self.embed_audio_tokens = nn.Embedding(
(config.num_codebooks * config.vocab_size), self.hidden_size
)
if self.audio_embed_size != self.hidden_size:
self.audio_embeds_projector = nn.Linear(
self.audio_embed_size, self.hidden_size, bias=False
)
else:
self.audio_embeds_projector = None
self.register_buffer(
"audio_tokens_offsets",
torch.arange(config.num_codebooks) * config.vocab_size,
persistent=False,
)
def forward(self, input_ids):
input_embeds = self.embed_audio_tokens(input_ids + self.audio_tokens_offsets)
if self.audio_embeds_projector:
input_embeds = self.audio_embeds_projector(input_embeds)
input_embeds = input_embeds.sum(dim=2)
return input_embeds
@auto_docstring
class BreezeBackboneModel(BreezePreTrainedModel):
def __init__(self, config):
super().__init__(config)
self.padding_idx = config.pad_token_id
self.vocab_size = config.vocab_size
self.embed_tokens = BreezeBackboneModelEmbeddings(config)
self.layers = nn.ModuleList(
[
BreezeDecoderLayer(config, layer_idx)
for layer_idx in range(config.num_hidden_layers)
]
)
self.norm = BreezeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.rotary_emb = BreezeRotaryEmbedding(config=config)
self.gradient_checkpointing = False
# Initialize weights and apply final processing
self.post_init()
@auto_docstring
def forward(
self,
input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
past_key_values: Cache | None = None,
inputs_embeds: torch.FloatTensor | None = None,
cache_position: torch.LongTensor | None = None,
use_cache: bool | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> BaseModelOutputWithPast:
r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`):
1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input
requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens.
2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users.
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
[`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
"""
if (input_ids is None) ^ (inputs_embeds is not None):
raise ValueError(
"You must specify exactly one of input_ids or inputs_embeds"
)
if inputs_embeds is None:
inputs_embeds: torch.Tensor = self.embed_tokens(input_ids)
if use_cache and past_key_values is None:
past_key_values = DynamicCache(config=self.config)
if cache_position is None:
past_seen_tokens = (
past_key_values.get_seq_length() if past_key_values is not None else 0
)
cache_position: torch.Tensor = torch.arange(
past_seen_tokens,
past_seen_tokens + inputs_embeds.shape[1],
device=inputs_embeds.device,
)
if position_ids is None:
position_ids = cache_position.unsqueeze(0)
causal_mask = create_causal_mask(
config=self.config,
input_embeds=inputs_embeds,
attention_mask=attention_mask,
cache_position=cache_position,
past_key_values=past_key_values,
position_ids=position_ids,
)
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids)
for decoder_layer in self.layers[: self.config.num_hidden_layers]:
hidden_states = decoder_layer(
hidden_states,
attention_mask=causal_mask,
position_ids=position_ids,
past_key_values=past_key_values,
cache_position=cache_position,
position_embeddings=position_embeddings,
**kwargs,
)
hidden_states = self.norm(hidden_states)
return BaseModelOutputWithPast(
last_hidden_state=hidden_states,
past_key_values=past_key_values,
)
from transformers import AutoConfig, PretrainedConfig
@auto_docstring(
custom_intro="""
The Breeze model consists of two llama-like auto-regressive transformer models: a backbone model that predicts the first codebook token and a depth decoder that predicts the other codebook tokens.
"""
)
class BreezeForConditionalGeneration(BreezePreTrainedModel, BreezeGenerationMixin):
_tied_weights_keys = [
"backbone_model.embed_tokens.embed_audio_tokens.weight",
"depth_decoder.model.embed_tokens.weight",
]
def __init__(self, config):
super().__init__(config)
self.vocab_size = config.vocab_size
# Extra backbone EOS class at index vocab_size; audio codebook ids remain [0, vocab_size).
self.backbone_eos_token_id = config.vocab_size
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size + 1, bias=False)
# embed_text_tokens needs to include audio special tokens (audio_token_id, audio_eos_token_id)
# since they appear in input_ids before being replaced with audio embeddings
# text_vocab_size already includes these tokens after tokenizer extension
self.embed_text_tokens = nn.Embedding(
config.text_vocab_size, config.hidden_size
)
# Use factory to create backbone (supports breeze/qwen3/llama etc.)
from .breeze_backbone_factory import BreezeBackboneFactory
self.backbone_model = BreezeBackboneFactory.create_backbone(config)
self.depth_decoder = BreezeDepthDecoderForCausalLM._from_config(
config.depth_decoder_config
)
self.codec_model = AutoModel.from_config(config.codec_config)
# Codec model is always in eval mode (not trainable)
self.codec_model.eval()
for param in self.codec_model.parameters():
param.requires_grad = False
self.text_encoder_trainable = False
text_encoder_config = getattr(config, "text_encoder_config", None)
if text_encoder_config is None:
self.text_encoder = None
self.text_encoder_proj = None
elif isinstance(text_encoder_config, dict):
config.text_encoder_config = AutoConfig.for_model(**text_encoder_config)
elif isinstance(config.text_encoder_config, PretrainedConfig):
config.text_encoder_config = text_encoder_config
else:
raise ValueError(
"text_encoder_config must be None, dict or PretrainedConfig instance"
)
if getattr(config, "text_encoder_config", None) is not None:
self.text_encoder_layer_projs = None
self.text_encoder_dimfusion_layer_start_idx = getattr(
config, "text_encoder_dimfusion_layer_start_idx", 1
) # 1 feature indicates from layer 1, feature 0 is embedding result
self.text_encoder_dimfusion_layer_end_idx = getattr(
config, "text_encoder_dimfusion_layer_end_idx", None
) # None indicates up to last layer
self.text_encoder_dimfusion_fuse_first_layer = getattr(
config, "text_encoder_dimfusion_fuse_first_layer", False
)
self.text_encoder_feature_layer_idx = getattr(
config, "text_encoder_feature_layer_idx", -1
)
if isinstance(self.text_encoder_feature_layer_idx, int):
self.text_encoder_feature_layer_idx = (
self.text_encoder_feature_layer_idx,
)
if self.text_encoder_feature_layer_idx != (-1,):
print(
f" - [BreezeForConditionalGeneration] Text encoder feature layer idx: {self.text_encoder_feature_layer_idx}"
)
text_encoder_attn_implementation = getattr(
config.text_encoder_config,
"preferred_attn_implementation",
None,
)
# Force eager attention for environments without flash_attn
if text_encoder_attn_implementation in (None, "flash_attention_2"):
text_encoder_attn_implementation = "eager"
config.text_encoder_config._attn_implementation = (
text_encoder_attn_implementation
)
# Skip random weight initialization since we'll load pretrained weights later
from transformers.modeling_utils import no_init_weights
with no_init_weights():
self.text_encoder = AutoModel.from_config(
config.text_encoder_config,
attn_implementation=text_encoder_attn_implementation,
dtype=torch.bfloat16,
)
text_encoder_proj_type = getattr(config, "text_encoder_proj_type", "linear")
if text_encoder_proj_type == "linear":
self.text_encoder_proj = nn.Linear(
config.text_encoder_config.hidden_size,
config.hidden_size,
bias=False,
)
elif text_encoder_proj_type == "mlp":
self.text_encoder_proj = nn.Sequential(
nn.Linear(
config.text_encoder_config.hidden_size,
config.hidden_size * 2,
bias=False,
),
nn.GELU(),
nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False),
)
elif text_encoder_proj_type == "breeze_text_encoder_adapter":
from .t5_adapter import BreezeTextEncoderAdapter
self.text_encoder_proj = BreezeTextEncoderAdapter(config=config)
elif text_encoder_proj_type == "breeze_dimfusion":
proj_out_hidden_size = config.hidden_size
if self.text_encoder_dimfusion_fuse_first_layer:
proj_out_hidden_size //= 2
self.text_encoder_proj = nn.Linear(
config.text_encoder_config.hidden_size
* len(self.text_encoder_feature_layer_idx),
proj_out_hidden_size,
bias=True, # bias=True, follow DimFusion design: https://github.com/huggingface/diffusers/blob/d0ae34d3136f7cb119a66fe0b9d526202343674c/src/diffusers/models/transformers/transformer_bria_fibo.py#L477
)
self.text_encoder_layer_projs = nn.ModuleList(
[
nn.Linear(
config.text_encoder_config.hidden_size,
config.hidden_size // 2,
bias=False,
)
for _ in range(self.config.num_hidden_layers)
]
)
dimfusion_kwargs = {
"text_encoder_dimfusion_layer_start_idx": self.text_encoder_dimfusion_layer_start_idx,
"text_encoder_dimfusion_layer_end_idx": self.text_encoder_dimfusion_layer_end_idx,
"text_encoder_dimfusion_fuse_first_layer": self.text_encoder_dimfusion_fuse_first_layer,
}
print(
" - [BreezeForConditionalGeneration] Text encoder DimFusion layer projections created."
)
for k, v in dimfusion_kwargs.items():
print(f" - {k}: {v}")
else:
raise ValueError(
f"Unsupported text_encoder_proj_type: {text_encoder_proj_type}"
)
print(
" - [BreezeForConditionalGeneration] Text encoder projection type:",
text_encoder_proj_type,
)
requires_grad = getattr(config.text_encoder_config, "requires_grad", False)
self.text_encoder_trainable = bool(requires_grad)
if not self.text_encoder_trainable:
nonzero_dropout_fields = self._get_nonzero_text_encoder_dropout_fields(
config.text_encoder_config
)
if nonzero_dropout_fields:
formatted_fields = ", ".join(
f"{name}={value}" for name, value in nonzero_dropout_fields
)
raise ValueError(
"Frozen text_encoder must be deterministic, but its config has non-zero dropout fields: "
f"{formatted_fields}. Set train_text_encoder=true or use a text encoder config with dropout=0."
)
self.text_encoder.eval()
for param in self.text_encoder.parameters():
param.requires_grad = requires_grad
import os
if os.environ.get("RANK", "0") == "0":
logger.info(
f"model.text_encoder initialized with requires_grad={requires_grad}"
)
# Disable num_items_in_batch to avoid incorrect loss normalization
# for depth decoder (which operates on different token counts than backbone)
self.accepts_loss_kwargs = False
# Temporarily remove text_encoder before post_init to avoid slow re-initialization
# (text_encoder weights will be loaded from pretrained checkpoint later)
_text_encoder = self.text_encoder
self.text_encoder = None
self.post_init()
self.text_encoder = _text_encoder
def get_input_embeddings(self):
return self.backbone_model.embed_tokens
def set_input_embeddings(self, value):
self.backbone_model.embed_tokens = value
def _tie_weights(self):
if self.config.tie_codebooks_embeddings:
self._tie_or_clone_weights(
self.backbone_model.embed_tokens.embed_audio_tokens,
self.depth_decoder.model.embed_tokens,
)
@staticmethod
def _get_nonzero_text_encoder_dropout_fields(text_encoder_config):
dropout_field_names = (
"dropout",
"dropout_rate",
"attention_dropout",
"hidden_dropout",
"hidden_dropout_prob",
"activation_dropout",
"classifier_dropout",
"classifier_dropout_rate",
"embd_pdrop",
"resid_pdrop",
)
nonzero_fields = []
for field_name in dropout_field_names:
value = getattr(text_encoder_config, field_name, None)
if value is None:
continue
try:
numeric_value = float(value)
except (TypeError, ValueError):
continue
if numeric_value != 0.0:
nonzero_fields.append((field_name, value))
return nonzero_fields
def train(self, mode: bool = True):
"""Override train() to keep non-trainable submodules in eval mode."""
super().train(mode)
# Always keep codec_model in eval mode
self.codec_model.eval()
if getattr(self, "text_encoder", None) is not None and not getattr(
self, "text_encoder_trainable", False
):
self.text_encoder.eval()
return self
@classmethod
def from_pretrained(cls, *args, **kwargs):
if kwargs.get("output_loading_info", False):
model, loading_info = super().from_pretrained(*args, **kwargs)
else:
model = super().from_pretrained(*args, **kwargs)
# copy depth decoder generation conf attr to the depth decoder generation config
prefix = "depth_decoder_"
prefix_len = len(prefix)
depth_decoder_attrs = {
attr[prefix_len:]: value
for attr, value in vars(model.generation_config).items()
if attr.startswith(prefix)
}
vars(model.depth_decoder.generation_config).update(
{"_from_model_config": False, **depth_decoder_attrs}
)
# remove the depth decoder generation conf attr from the model generation config
for attr in depth_decoder_attrs:
delattr(model.generation_config, prefix + attr)
if "output_loading_info" in kwargs:
return model, loading_info
else:
return model
def save_pretrained(self, *args, **kwargs):
# copy the depth decoder generation config attributes to the model generation config
prefix = "depth_decoder_"
depth_decoder_attrs = self.depth_decoder.generation_config.to_diff_dict()
depth_decoder_attrs.pop("transformers_version", None)
for attr, value in depth_decoder_attrs.items():
setattr(self.generation_config, prefix + attr, value)
super().save_pretrained(*args, **kwargs)
def gradient_checkpointing_enable(self, *args, **kwargs):
gradient_checkpointing_kwargs = kwargs.pop("gradient_checkpointing_kwargs", {})
sub_modules_to_disable_default = [
# 'codec_model',
# # 'backbone_model',
# 'text_encoder',
# 'text_encoder_proj',
# 'depth_decoder'
]
if gradient_checkpointing_kwargs is None:
gradient_checkpointing_kwargs = {}
sub_modules_to_disable = gradient_checkpointing_kwargs.pop(
"sub_modules_to_disable", sub_modules_to_disable_default
)
# call super first to enable gradient checkpointing
ret = super().gradient_checkpointing_enable(*args, **kwargs)
# override sub-modules to disable gradient checkpointing
sub_modules_to_disable = gradient_checkpointing_kwargs.get(
"sub_modules_to_disable", sub_modules_to_disable
)
# 1. disable gradient checkpointing for specified sub-modules
import os
rank = int(os.environ.get("RANK", "0"))
print0 = lambda *args, **kwargs: print(*args, **kwargs) if rank == 0 else None
for module_name in sub_modules_to_disable:
module = getattr(self, module_name, None)
module_layers = (
[
(name, mod)
for name, mod in module.named_modules()
if isinstance(mod, GradientCheckpointingLayer)
]
if module is not None
else []
)
if not module_layers:
print0(
f" - [BreezeForConditionalGeneration] gradient_checkpointing_disable skipped for sub-module: {module_name}"
)
continue
print0(
f" - [BreezeForConditionalGeneration] gradient_checkpointing_disable applied for sub-module: {module_name}"
)
for layer_name, layer in module_layers:
layer.gradient_checkpointing = False
print0(
f" - Disabled gradient checkpointing for layer: {module_name}.{layer_name}"
)
return ret
def _batched_text_encoder_forward(self, segments, output_hidden_states=False):
"""Run text encoder on a list of variable-length token ID tensors using padded batching.
Args:
segments: list of 1D tensors, each (seg_len,)
Returns:
hidden_states: list of 2D tensors, each (seg_len, hidden) with padding removed
layer_hidden_states: list of lists, per-segment layer hidden states (each (seg_len, hidden)),
or None when not requested.
"""
if not segments:
return [], []
if (
getattr(self, "_fast_text_encoder_cudagraph", False)
and not output_hidden_states
):
from .text_encoder_graph import TextEncoderGraphCache
cache = getattr(self, "_fast_text_encoder_graph_cache", None)
if cache is None:
cache = TextEncoderGraphCache(self.text_encoder, token_granularity=32)
self._fast_text_encoder_graph_cache = cache
return cache(segments)
device = segments[0].device
lengths = [s.shape[0] for s in segments]
feature_layer_idx = self.text_encoder_feature_layer_idx
if isinstance(feature_layer_idx, int):
feature_layer_idx = (feature_layer_idx,)
elif isinstance(feature_layer_idx, list):
feature_layer_idx = tuple(feature_layer_idx)
sorted_indices = sorted(range(len(segments)), key=lambda i: lengths[i])
buckets = []
current_bucket = []
current_min_len = None
for idx in sorted_indices:
length = lengths[idx]
if not current_bucket:
current_bucket = [idx]
current_min_len = length
elif length / max(current_min_len, 1) <= 2:
current_bucket.append(idx)
else:
buckets.append(current_bucket)
current_bucket = [idx]
current_min_len = length
if current_bucket:
buckets.append(current_bucket)
hidden_states = [None] * len(segments)
layer_hidden_states = [None] * len(segments) if output_hidden_states else None
for bucket_indices in buckets:
bucket_lengths = [lengths[i] for i in bucket_indices]
max_len = max(bucket_lengths)
padded_ids = torch.zeros(
len(bucket_indices), max_len, dtype=segments[0].dtype, device=device
)
attn_mask = torch.zeros(
len(bucket_indices), max_len, dtype=torch.long, device=device
)
pos_ids = torch.zeros(
len(bucket_indices), max_len, dtype=torch.long, device=device
)
for bucket_pos, idx in enumerate(bucket_indices):
length = lengths[idx]
padded_ids[bucket_pos, :length] = segments[idx]
attn_mask[bucket_pos, :length] = 1
pos_ids[bucket_pos, :length] = torch.arange(length, device=device)
output = self.text_encoder(
input_ids=padded_ids,
attention_mask=attn_mask,
position_ids=pos_ids,
output_hidden_states=output_hidden_states,
)
if feature_layer_idx == (-1,):
full_hs = output.last_hidden_state
elif isinstance(feature_layer_idx, tuple):
if output.hidden_states is None:
raise ValueError(
"output_hidden_states must be enabled when selecting non-final text encoder layers"
)
full_hs = torch.concat(
[output.hidden_states[li] for li in feature_layer_idx], dim=-1
)
else:
raise ValueError(
f"Unsupported text_encoder_feature_layer_idx: {feature_layer_idx}"
)
for bucket_pos, idx in enumerate(bucket_indices):
hidden_states[idx] = full_hs[bucket_pos, : lengths[idx]]
if output.hidden_states is not None:
all_layer_hs = list(output.hidden_states)
if layer_hidden_states is None:
layer_hidden_states = [None] * len(segments)
for bucket_pos, idx in enumerate(bucket_indices):
layer_hidden_states[idx] = [
lhs[bucket_pos, : lengths[idx]] for lhs in all_layer_hs
]
return hidden_states, layer_hidden_states
def convert_input_ids_to_embeds(
self,
input_ids: torch.Tensor | None = None,
text_ids_mask: torch.Tensor | None = None,
text_ids_len: torch.Tensor | None = None,
attention_mask: torch.Tensor | None = None,
):
assert self.text_encoder is not None, (
"text_encoder is None, cannot convert input_ids to embeds"
)
assert text_ids_mask is not None, (
"text_ids_mask is None, cannot convert input_ids to embeds"
)
assert text_ids_len is not None, (
"text_ids_len is None, cannot convert input_ids to embeds"
)
# Ensure text_ids_len matches mask-derived count (helps catch bad batching)
total_text_tokens = int(text_ids_mask.sum().item())
assert total_text_tokens == int(text_ids_len.sum().item()), (
f"text_ids_mask sum {total_text_tokens} does not match text_ids_len sum {int(text_ids_len.sum().item())}"
)
text_ids_len_list = [int(x) for x in text_ids_len.reshape(-1).tolist()]
text_ids_len_idx = 0
# == Phase 1: Parse all samples and collect segments ==
# Keep row-major order so the projected embeddings line up with
# boolean assignment into inputs_embeds[text_ids_mask].
all_segment_lengths = []
all_seg_token_ids = []
for batch_idx in range(input_ids.shape[0]):
mask_row = text_ids_mask[batch_idx]
sample_text_tokens = int(mask_row.sum().item())
if sample_text_tokens == 0:
continue
text_ids = input_ids[batch_idx][mask_row] # (total_text_len,)
segment_lengths = []
running = 0
while running < sample_text_tokens:
if text_ids_len_idx >= len(text_ids_len_list):
raise ValueError(
"text_ids_len exhausted before covering all text tokens in the batch"
)
seg_len = text_ids_len_list[text_ids_len_idx]
text_ids_len_idx += 1
if seg_len <= 0:
continue
segment_lengths.append(seg_len)
running += seg_len
if running != sample_text_tokens:
raise ValueError(
f"text_ids_len segments sum {running} does not match text_ids_mask sum {sample_text_tokens} for batch index {batch_idx}"
)
seg_token_ids = text_ids.split(segment_lengths, dim=0) # list of 1D tensors
all_segment_lengths.extend(segment_lengths)
all_seg_token_ids.extend(seg_token_ids)
if text_ids_len_idx != len(text_ids_len_list):
raise ValueError(
f"Unused text_ids_len entries detected: consumed {text_ids_len_idx} of {len(text_ids_len_list)}"
)
# == Phase 2: Text encoder forward ==
# Batched independent encoding: each text segment is its own row in
# the padded text-encoder batch. This prevents cross-sample and
# cross-segment attention contamination under the flattening collator.
if all_seg_token_ids:
feature_layer_idx = self.text_encoder_feature_layer_idx
if isinstance(feature_layer_idx, int):
feature_layer_idx = (feature_layer_idx,)
elif isinstance(feature_layer_idx, list):
feature_layer_idx = tuple(feature_layer_idx)
needs_layer_hidden_states = (
self.text_encoder_layer_projs is not None or feature_layer_idx != (-1,)
)
seg_hidden_states, seg_layer_hidden_states = (
self._batched_text_encoder_forward(
all_seg_token_ids,
output_hidden_states=needs_layer_hidden_states,
)
)
seg_hidden_states = [hs.unsqueeze(0) for hs in seg_hidden_states]
text_embeds, text_encoder_layer_hidden_states = self._project_segments(
all_segment_lengths,
seg_hidden_states,
seg_layer_hidden_states,
is_separate=True,
)
else:
text_embeds = torch.empty(
(0, self.config.hidden_size),
device=input_ids.device,
dtype=self.embed_text_tokens.weight.dtype,
)
text_encoder_layer_hidden_states = None
# == Phase 3: Assemble final outputs ==
# Create inputs_embeds tensor
inputs_embeds = torch.zeros(
(input_ids.shape[0], input_ids.shape[1], self.config.hidden_size),
dtype=text_embeds.dtype,
device=text_embeds.device,
)
inputs_embeds[text_ids_mask] = text_embeds
return inputs_embeds, text_encoder_layer_hidden_states
def _project_segments(
self, segment_lengths, seg_hidden_states, seg_layer_hidden_states, is_separate
):
"""Project text encoder hidden states per segment and handle layer projections.
Args:
segment_lengths: list of int, length of each segment
seg_hidden_states: list of (1, seg_len, hidden) tensors
seg_layer_hidden_states: list of layer hidden state lists per segment.
- If is_separate=True: each entry is a list of (seg_len, hidden) tensors (one per layer).
- If is_separate=False: each entry is a list of (1, total_len, hidden) tensors (shared across segments).
is_separate: whether segments were encoded separately
Returns:
text_embeds: (total_text_len, hidden) projected embeddings
text_encoder_layer_hidden_states: list of (total_text_len, hidden) or None
"""
projected_segments = []
for seg_idx, seg_hs in enumerate(seg_hidden_states):
proj = self.text_encoder_proj
if isinstance(proj, (nn.Linear, nn.Sequential)):
projected_segments.append(proj(seg_hs).squeeze(0))
else:
seg_pos = torch.arange(seg_hs.shape[1], device=seg_hs.device).unsqueeze(
0
)
projected_segments.append(
proj(inputs_embeds=seg_hs, position_ids=seg_pos).squeeze(0)
)
text_embeds = torch.cat(projected_segments, dim=0)
if self.text_encoder_layer_projs is not None:
num_layers = len(self.text_encoder_layer_projs)
projected_layer_hidden_states = []
for layer_idx in range(num_layers):
layer_parts = []
for seg_idx, seg_layer_hs_list in enumerate(seg_layer_hidden_states):
selected = seg_layer_hs_list[
self.text_encoder_dimfusion_layer_start_idx : self.text_encoder_dimfusion_layer_end_idx
]
if len(selected) < num_layers:
selected = selected + [selected[-1]] * (
num_layers - len(selected)
)
elif len(selected) > num_layers:
selected = selected[-num_layers:]
seg_layer_hs = selected[layer_idx]
if is_separate:
# Already per-segment: (seg_len, hidden) -> (1, seg_len, hidden)
seg_layer_hs = seg_layer_hs.unsqueeze(0)
else:
# Joint encoding: (1, total_len, hidden) -> split -> (1, seg_len, hidden)
seg_layer_hs = (
seg_layer_hs.squeeze(0)
.split(segment_lengths, dim=0)[seg_idx]
.unsqueeze(0)
)
layer_proj = self.text_encoder_layer_projs[layer_idx]
layer_parts.append(layer_proj(seg_layer_hs).squeeze(0))
projected_layer_hidden_states.append(torch.cat(layer_parts, dim=0))
text_encoder_layer_hidden_states = projected_layer_hidden_states
else:
text_encoder_layer_hidden_states = None
if self.text_encoder_dimfusion_fuse_first_layer:
assert text_encoder_layer_hidden_states is not None, (
"text_encoder_layer_hidden_states is None, cannot fuse first layer hidden states"
)
layer0_hidden_states = text_encoder_layer_hidden_states[0]
assert (
layer0_hidden_states.shape[-1] + text_embeds.shape[-1]
) == self.config.hidden_size, (
f"Dimension mismatch when fusing layer 0 hidden states with text embeddings, but got {layer0_hidden_states.shape[-1]} + {text_embeds.shape[-1]} != {self.config.hidden_size}"
)
text_embeds = torch.cat([text_embeds, layer0_hidden_states], dim=-1)
return text_embeds, text_encoder_layer_hidden_states
def _merge_input_ids_with_input_values(
self,
input_ids: torch.Tensor | None = None,
input_values: torch.Tensor | None = None,
labels: torch.Tensor | None = None,
text_ids_mask: torch.Tensor | None = None,
text_ids_len: torch.Tensor | None = None,
attention_mask: torch.Tensor | None = None,
) -> torch.Tensor | None:
"""
Merges the input_ids and input_values to produce a single inputs_embeds tensor:
1 - Uses pre-encoded codebook tokens from input_values (no codec inference needed).
2 - Embeds codebook tokens and places them at the correct positions in the inputs_embeds tensor.
3 - If labels are provided, expands them to match codebook dimensions and position the target codebook tokens in the inputs_embeds tensor.
Args:
input_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`):
The input ids to embed.
input_values (`torch.Tensor` of shape `(batch_size, audio_sequence_length, num_codebooks)`):
The pre-encoded audio tokens (RVQ codebook indices).
labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the loss.
"""
text_encoder_layer_hidden_states = None
if self.text_encoder is not None:
inputs_embeds, text_encoder_layer_hidden_states = (
self.convert_input_ids_to_embeds(
input_ids,
text_ids_mask,
text_ids_len,
attention_mask=attention_mask,
)
)
else:
inputs_embeds = self.embed_text_tokens(input_ids)
if input_values is not None:
# input_values are already pre-encoded tokens: (batch_size, audio_seq_len, num_codebooks)
batched_audio_token_ids = input_values
# Get mask for audio token positions in input_ids
audio_token_id = self.config.audio_token_id
audio_token_mask = input_ids == audio_token_id
# Debug: check if audio tokens are found
num_audio_tokens = audio_token_mask.sum().item()
if num_audio_tokens == 0 and input_values is not None:
# This should not happen - we have audio data but no audio token markers
unique_ids = torch.unique(input_ids)
raise RuntimeError(
f"Audio token mismatch: Expected audio_token_id={audio_token_id} in input_ids, "
f"but found 0 matches. input_values shape: {input_values.shape}. "
f"Unique input_ids: {unique_ids.tolist()[:20]}... (showing first 20)"
)
# Convert patches tokens to embeddings
audio_embeds = self.backbone_model.embed_tokens(batched_audio_token_ids)
# Flatten audio_embeds for assignment to masked positions
# audio_embeds shape: (batch_size, audio_seq_len, hidden_size)
# Need to flatten to match the flattened audio_token_mask
audio_embeds_flat = audio_embeds.reshape(-1, audio_embeds.shape[-1])
# Ensure dtype matches
audio_embeds_flat = audio_embeds_flat.to(inputs_embeds.dtype)
inputs_embeds[audio_token_mask] = audio_embeds_flat
# same for the audio eos token.
audio_eos_frame_ids = (
torch.ones(
(1, 1, self.config.num_codebooks),
device=input_ids.device,
dtype=torch.long,
)
* self.config.codebook_eos_token_id
)
audio_eos_embeds = self.backbone_model.embed_tokens(
audio_eos_frame_ids
).squeeze(1)
# Ensure dtype matches
audio_eos_embeds = audio_eos_embeds.to(inputs_embeds.dtype)
audio_eos_token_mask = input_ids == self.config.audio_eos_token_id
inputs_embeds[audio_eos_token_mask] = audio_eos_embeds.repeat(
audio_eos_token_mask.sum(), 1
)
# if the labels are provided, we need to expand the labels to (batch_size, seq_length, num_codebooks)
if labels is not None:
labels_expanded = labels.unsqueeze(-1).repeat(
1, 1, self.config.num_codebooks
)
# Flatten batched_audio_token_ids for assignment to masked positions
batched_audio_token_ids_flat = batched_audio_token_ids.reshape(
-1, batched_audio_token_ids.shape[-1]
)
labels_expanded[audio_token_mask] = batched_audio_token_ids_flat
labels_expanded[audio_eos_token_mask] = -100
eos_positions = audio_eos_token_mask.nonzero(as_tuple=True)
labels_expanded[eos_positions[0], eos_positions[1], 0] = (
self.backbone_eos_token_id
)
# mask depth decoder
depth_decoder_ignore_frames_mask = labels == -101
depth_decoder_ignore_frames_mask &= ~audio_eos_token_mask
depth_decoder_ignore_frames_idxs = (
depth_decoder_ignore_frames_mask.nonzero(as_tuple=True)
)
# set the first codebook patch to -100 to ignore
labels_expanded[
depth_decoder_ignore_frames_idxs[0],
depth_decoder_ignore_frames_idxs[1],
1 : self.config.num_codebooks,
] = -100
labels = labels_expanded
return {
"inputs_embeds": inputs_embeds,
"labels": labels,
"text_encoder_layer_hidden_states": text_encoder_layer_hidden_states,
"text_ids_mask": text_ids_mask,
}
def _get_audio_token_from_batch(self, audio_batch):
with torch.no_grad():
print("Encoding audio batch of shape:", audio_batch.shape)
codec_outputs = self.codec_model.encode(audio_batch.unsqueeze(0))
codebook_ids = codec_outputs.audio_codes.transpose(1, -1)[0]
return codebook_ids
def prepare_inputs_for_generation(
self,
input_ids: torch.LongTensor,
past_key_values: Cache | None = None,
attention_mask: torch.LongTensor | None = None,
inputs_embeds: torch.FloatTensor | None = None,
cache_position: torch.LongTensor | None = None,
**kwargs,
):
model_inputs = super().prepare_inputs_for_generation(
input_ids=input_ids,
past_key_values=past_key_values,
attention_mask=attention_mask,
inputs_embeds=inputs_embeds,
cache_position=cache_position,
**kwargs,
)
# if input_ids is not None and input_ids.ndim == 2 and model_inputs.get("inputs_embeds") is None:
# print("------- Merging input_ids with input_values for generation...")
# merged_inputs = self._merge_input_ids_with_input_values(
# input_ids=input_ids,
# input_values=kwargs.get("input_values"),
# labels=kwargs.get("labels"),
# )
# model_inputs.update(
# {"inputs_embeds": merged_inputs["inputs_embeds"], "labels": merged_inputs["labels"], "input_ids": None}
# )
# cache_position = model_inputs["cache_position"]
# is_prefill = cache_position is not None and cache_position[0] == 0
# if is_prefill:
# pass
# else:
# # Remove text_encoder_layer_hidden_states to indicate no fusion needed during decoding
# model_inputs.pop("text_encoder_layer_hidden_states", None)
return model_inputs
@can_return_tuple
@auto_docstring
def forward(
self,
input_ids: torch.LongTensor = None,
input_values: torch.Tensor | None = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
past_key_values: Cache | list[torch.FloatTensor] | None = None,
inputs_embeds: torch.FloatTensor | None = None,
labels: torch.LongTensor | None = None,
use_cache: bool | None = None,
cache_position: torch.LongTensor | None = None,
logits_to_keep: int | torch.Tensor = 0,
text_ids_mask: torch.Tensor | None = None,
text_ids_len: torch.Tensor | None = None,
**kwargs: Unpack[TransformersKwargs],
) -> tuple | BreezeOutputWithPast:
r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`):
1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input
requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens.
2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users.
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
[`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the masked language modeling loss. Indices should be in `[config.audio_token_id, -100, -101]`.
Requires targeted `input_values` to be provided as audio tokens will be inferred from it using the `codec_model`.
- `config.audio_token_id` indicates an audio frames (considering sequence length elements as frames)
- `-100` will be ignored in the loss computation
- `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels)
Such labels can be prepared using `output_labels=True` when calling [`the repository input preparation utilities`].
logits_to_keep (`int` or `torch.Tensor`, *optional*):
Kept for compatibility. Does not support another value than:
1. `0`, which is equivalent to keeping all logits, used in the training regime
2. `1`, which is equivalent to keeping only the last logit, used in the generation regime
text_ids_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
Marks text-token positions consumed by the text encoder.
text_ids_len (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Number of valid text tokens for each item in the batch.
"""
text_encoder_layer_hidden_states = None
if input_ids is not None and input_ids.ndim == 2:
merged_inputs = self._merge_input_ids_with_input_values(
input_ids,
input_values,
labels,
text_ids_mask=text_ids_mask,
text_ids_len=text_ids_len,
attention_mask=attention_mask,
)
inputs_embeds = merged_inputs["inputs_embeds"]
labels = merged_inputs["labels"]
input_ids = None
text_encoder_layer_hidden_states = merged_inputs.get(
"text_encoder_layer_hidden_states", None
)
backbone_outputs = self.backbone_model(
input_ids=input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=past_key_values,
inputs_embeds=inputs_embeds,
use_cache=use_cache,
cache_position=cache_position,
text_encoder_layer_hidden_states=text_encoder_layer_hidden_states,
text_ids_mask=text_ids_mask,
**kwargs,
)
backbone_hidden_states = backbone_outputs[0]
backbone_hidden_states_for_depth_decoder = backbone_hidden_states
backbone_all_hidden_states = backbone_outputs.hidden_states
if backbone_all_hidden_states is None:
backbone_all_hidden_states = (backbone_hidden_states_for_depth_decoder,)
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
slice_indices = (
slice(-logits_to_keep, None)
if isinstance(logits_to_keep, int)
else logits_to_keep
)
loss = None
backbone_loss = None
backbone_logits = None
depth_decoder_loss = None
depth_decoder_outputs = None
if labels is not None:
# Filter out num_items_in_batch from kwargs to ensure reduction='mean' is used
# This prevents incorrect loss normalization when backbone and depth decoder
# operate on different numbers of tokens
loss_kwargs = {k: v for k, v in kwargs.items() if k != "num_items_in_batch"}
# select first codebook as labels for the backbone model
backbone_labels = labels[:, :, 0]
backbone_logits = self.lm_head(backbone_hidden_states[:, slice_indices, :])
backbone_loss = self.loss_function(
logits=backbone_logits,
labels=backbone_labels,
vocab_size=self.lm_head.out_features,
**loss_kwargs,
)
# Compute logits only when actually needed
if backbone_logits is None and labels is None:
backbone_logits = self.lm_head(backbone_hidden_states[:, slice_indices, :])
if labels is not None:
# for the depth decoder, we need to select the frames to train on
# those are frames where the label is not uniformly `ignore_index` along the codebook dimension
train_mask = ~(labels[:, :, 1:] == -100).all(dim=-1)
depth_decoder_input_ids = labels[train_mask][
..., : self.config.num_codebooks - 1
]
# add place holder in position 0 that will be replaced by the backbone_last_hidden_state
depth_decoder_input_ids = nn.functional.pad(
depth_decoder_input_ids, (1, 0), value=0
)
train_idxs = train_mask.nonzero(as_tuple=True)
backbone_last_hidden_states = backbone_hidden_states_for_depth_decoder[
train_idxs[0], train_idxs[1] - 1, :
]
depth_decoder_labels = labels[train_mask]
depth_decoder_outputs = self.depth_decoder(
input_ids=depth_decoder_input_ids,
backbone_last_hidden_state=backbone_last_hidden_states,
use_cache=use_cache,
return_dict=True,
labels=depth_decoder_labels,
**loss_kwargs,
)
depth_decoder_loss = depth_decoder_outputs.loss
if self.training:
loss = (
backbone_loss
+ depth_decoder_loss * self.config.depth_header_loss_weight
)
else:
loss = backbone_loss + depth_decoder_loss
depth_decoder_hidden_states = None
if depth_decoder_outputs is not None:
depth_decoder_hidden_states = depth_decoder_outputs.hidden_states
return BreezeOutputWithPast(
loss=loss,
backbone_loss=backbone_loss,
depth_decoder_loss=depth_decoder_loss,
logits=backbone_logits,
past_key_values=backbone_outputs.past_key_values,
hidden_states=backbone_all_hidden_states,
attentions=backbone_outputs.attentions,
depth_decoder_logits=depth_decoder_outputs.logits
if depth_decoder_outputs is not None
else None,
depth_decoder_past_key_values=depth_decoder_outputs.past_key_values
if depth_decoder_outputs is not None
else None,
depth_decoder_hidden_states=depth_decoder_hidden_states,
depth_decoder_attentions=depth_decoder_outputs.attentions
if depth_decoder_outputs is not None
else None,
)
__all__ = [
"BreezeBackboneModel",
"BreezeDepthDecoderForCausalLM",
"BreezeDepthDecoderModel",
"BreezeForConditionalGeneration",
"BreezePreTrainedModel",
]
AutoModel.register(BreezeConfig, BreezeForConditionalGeneration, exist_ok=True)
# Breeze model implementation adapted from Hugging Face Transformers.
#
# coding=utf-8
# Copyright 2025 Sesame and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
import torch
from torch import nn
from transformers.activations import ACT2FN
from transformers.cache_utils import Cache, DynamicCache
from transformers.generation import GenerationMixin
from transformers.integrations import use_kernel_forward_from_hub
from transformers.masking_utils import create_causal_mask
from transformers.modeling_layers import GradientCheckpointingLayer
from transformers.modeling_outputs import (
BaseModelOutputWithPast,
CausalLMOutputWithPast,
)
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
from transformers.models.auto import AutoModel
from transformers.processing_utils import Unpack
from transformers.utils import (
ModelOutput,
TransformersKwargs,
auto_docstring,
can_return_tuple,
logging,
)
from transformers.utils.deprecation import deprecate_kwarg
from .breeze_config import BreezeConfig, BreezeDepthDecoderConfig
from .generation_breeze import BreezeGenerationMixin
logger = logging.get_logger(__name__)
class BreezeOutputWithPast(ModelOutput):
r"""
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
`(batch_size, num_heads, sequence_length, embed_size_per_head)`)
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
depth_decoder_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
Language modeling loss (for next-token prediction) of the depth decoder model.
depth_decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the depth decoder (scores for each vocabulary token before SoftMax).
depth_decoder_past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
`(batch_size, num_heads, sequence_length, embed_size_per_head)`)
depth_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
depth_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
sequence_length)`.
backbone_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
Language modeling loss (for next-token prediction) of the backbone model.
"""
loss: torch.FloatTensor | None = None
logits: torch.FloatTensor = None
past_key_values: tuple[tuple[torch.FloatTensor]] | None = None
hidden_states: tuple[torch.FloatTensor, ...] | None = None
attentions: tuple[torch.FloatTensor, ...] | None = None
depth_decoder_loss: torch.FloatTensor | None = None
depth_decoder_logits: torch.FloatTensor = None
depth_decoder_past_key_values: tuple[tuple[torch.FloatTensor]] | None = None
depth_decoder_hidden_states: tuple[torch.FloatTensor, ...] | None = None