"""ECAPA-TDNN Speaker Encoder model. Standalone implementation of the ECAPA-TDNN speaker encoder extracted from Qwen3-TTS. Produces fixed-dimensional x-vector speaker embeddings from log-mel spectrogram input. Architecture: ECAPA-TDNN (Emphasized Channel Attention, Propagation and Aggregation in TDNN Based Speaker Verification) Paper: https://arxiv.org/abs/2005.07143 This file is self-contained and depends only on torch and transformers. """ import torch from torch import nn from torch.nn import functional as F from transformers import PreTrainedModel from transformers.modeling_outputs import BaseModelOutputWithNoAttention from .configuration_ecapa_tdnn import EcapaTdnnSpeakerEncoderConfig class TimeDelayNetBlock(nn.Module): """1-D convolution + ReLU (TDNN layer).""" def __init__(self, in_channels, out_channels, kernel_size, dilation): super().__init__() self.conv = nn.Conv1d( in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, dilation=dilation, padding="same", padding_mode="reflect", ) self.activation = nn.ReLU() def forward(self, hidden_states: torch.Tensor): return self.activation(self.conv(hidden_states)) class Res2NetBlock(nn.Module): """Multi-scale Res2Net block using TDNN sub-blocks.""" def __init__(self, in_channels, out_channels, scale=8, kernel_size=3, dilation=1): super().__init__() in_channel = in_channels // scale hidden_channel = out_channels // scale self.blocks = nn.ModuleList( [ TimeDelayNetBlock(in_channel, hidden_channel, kernel_size=kernel_size, dilation=dilation) for _ in range(scale - 1) ] ) self.scale = scale def forward(self, hidden_states): outputs = [] for i, hidden_part in enumerate(torch.chunk(hidden_states, self.scale, dim=1)): if i == 0: output_part = hidden_part elif i == 1: output_part = self.blocks[i - 1](hidden_part) else: output_part = self.blocks[i - 1](hidden_part + output_part) outputs.append(output_part) return torch.cat(outputs, dim=1) class SqueezeExcitationBlock(nn.Module): """Channel-wise squeeze-and-excitation attention.""" def __init__(self, in_channels, se_channels, out_channels): super().__init__() self.conv1 = nn.Conv1d( in_channels=in_channels, out_channels=se_channels, kernel_size=1, padding="same", padding_mode="reflect", ) self.relu = nn.ReLU(inplace=True) self.conv2 = nn.Conv1d( in_channels=se_channels, out_channels=out_channels, kernel_size=1, padding="same", padding_mode="reflect", ) self.sigmoid = nn.Sigmoid() def forward(self, hidden_states): hidden_states_mean = hidden_states.mean(dim=2, keepdim=True) hidden_states_mean = self.relu(self.conv1(hidden_states_mean)) hidden_states_mean = self.sigmoid(self.conv2(hidden_states_mean)) return hidden_states * hidden_states_mean class SqueezeExcitationRes2NetBlock(nn.Module): """ECAPA-TDNN building block: TDNN → Res2Net → TDNN → SE, with residual.""" def __init__(self, in_channels, out_channels, res2net_scale=8, se_channels=128, kernel_size=1, dilation=1): super().__init__() self.out_channels = out_channels self.tdnn1 = TimeDelayNetBlock(in_channels, out_channels, kernel_size=1, dilation=1) self.res2net_block = Res2NetBlock(out_channels, out_channels, res2net_scale, kernel_size, dilation) self.tdnn2 = TimeDelayNetBlock(out_channels, out_channels, kernel_size=1, dilation=1) self.se_block = SqueezeExcitationBlock(out_channels, se_channels, out_channels) def forward(self, hidden_state): residual = hidden_state hidden_state = self.tdnn1(hidden_state) hidden_state = self.res2net_block(hidden_state) hidden_state = self.tdnn2(hidden_state) hidden_state = self.se_block(hidden_state) return hidden_state + residual class AttentiveStatisticsPooling(nn.Module): """Attentive statistics pooling — produces concatenated weighted mean and std.""" def __init__(self, channels, attention_channels=128): super().__init__() self.eps = 1e-12 self.tdnn = TimeDelayNetBlock(channels * 3, attention_channels, 1, 1) self.tanh = nn.Tanh() self.conv = nn.Conv1d( in_channels=attention_channels, out_channels=channels, kernel_size=1, padding="same", padding_mode="reflect", ) @staticmethod def _length_to_mask(length, max_len=None, dtype=None, device=None): if max_len is None: max_len = length.max().long().item() mask = torch.arange(max_len, device=length.device, dtype=length.dtype).expand( len(length), max_len ) < length.unsqueeze(1) return torch.as_tensor(mask, dtype=dtype, device=device) def _compute_statistics(self, x, m, dim=2): mean = (m * x).sum(dim) std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(self.eps)) return mean, std def forward(self, hidden_states): seq_length = hidden_states.shape[-1] lengths = torch.ones(hidden_states.shape[0], device=hidden_states.device) mask = self._length_to_mask( lengths * seq_length, max_len=seq_length, dtype=hidden_states.dtype, device=hidden_states.device ) mask = mask.unsqueeze(1) total = mask.sum(dim=2, keepdim=True) mean, std = self._compute_statistics(hidden_states, mask / total) mean = mean.unsqueeze(2).repeat(1, 1, seq_length) std = std.unsqueeze(2).repeat(1, 1, seq_length) attention = torch.cat([hidden_states, mean, std], dim=1) attention = self.conv(self.tanh(self.tdnn(attention))) attention = attention.masked_fill(mask == 0, float("-inf")) attention = F.softmax(attention, dim=2) mean, std = self._compute_statistics(hidden_states, attention) pooled_stats = torch.cat((mean, std), dim=1) pooled_stats = pooled_stats.unsqueeze(2) return pooled_stats class EcapaTdnnSpeakerEncoderPreTrainedModel(PreTrainedModel): config_class = EcapaTdnnSpeakerEncoderConfig base_model_prefix = "encoder" def _init_weights(self, module): std = 0.02 if isinstance(module, (nn.Linear, nn.Conv1d)): module.weight.data.normal_(mean=0.0, std=std) if module.bias is not None: module.bias.data.zero_() class EcapaTdnnSpeakerEncoder(EcapaTdnnSpeakerEncoderPreTrainedModel): """ECAPA-TDNN speaker encoder. Takes a log-mel spectrogram of shape ``(batch, time, mel_dim)`` and returns a fixed-dimensional speaker embedding of shape ``(batch, enc_dim)``. This is a standalone extraction of the speaker encoder from Qwen3-TTS, compatible with the HuggingFace ``AutoModel`` API. """ def __init__(self, config: EcapaTdnnSpeakerEncoderConfig): super().__init__(config) if len(config.enc_channels) != len(config.enc_kernel_sizes) or len(config.enc_channels) != len( config.enc_dilations ): raise ValueError("enc_channels, enc_kernel_sizes and enc_dilations must have the same length") self.channels = config.enc_channels self.blocks = nn.ModuleList() # Initial TDNN layer self.blocks.append( TimeDelayNetBlock( config.mel_dim, config.enc_channels[0], config.enc_kernel_sizes[0], config.enc_dilations[0], ) ) # SE-Res2Net layers for i in range(1, len(config.enc_channels) - 1): self.blocks.append( SqueezeExcitationRes2NetBlock( config.enc_channels[i - 1], config.enc_channels[i], res2net_scale=config.enc_res2net_scale, se_channels=config.enc_se_channels, kernel_size=config.enc_kernel_sizes[i], dilation=config.enc_dilations[i], ) ) # Multi-layer feature aggregation self.mfa = TimeDelayNetBlock( config.enc_channels[-1], config.enc_channels[-1], config.enc_kernel_sizes[-1], config.enc_dilations[-1], ) # Attentive Statistical Pooling self.asp = AttentiveStatisticsPooling( config.enc_channels[-1], attention_channels=config.enc_attention_channels, ) # Final linear transformation self.fc = nn.Conv1d( in_channels=config.enc_channels[-1] * 2, out_channels=config.enc_dim, kernel_size=1, padding="same", padding_mode="reflect", ) self.post_init() def forward(self, input_values=None, **kwargs): """ Args: input_values: Log-mel spectrogram tensor of shape ``(batch, time, mel_dim)``. Returns: ``BaseModelOutputWithNoAttention`` with ``last_hidden_state`` of shape ``(batch, enc_dim)``. """ hidden_states = input_values # Transpose to (batch, channels, time) for Conv1d hidden_states = hidden_states.transpose(1, 2) hidden_states_list = [] for layer in self.blocks: hidden_states = layer(hidden_states) hidden_states_list.append(hidden_states) # Multi-layer feature aggregation hidden_states = torch.cat(hidden_states_list[1:], dim=1) hidden_states = self.mfa(hidden_states) # Attentive Statistical Pooling hidden_states = self.asp(hidden_states) # Final linear transformation hidden_states = self.fc(hidden_states) hidden_states = hidden_states.squeeze(-1) return BaseModelOutputWithNoAttention(last_hidden_state=hidden_states)