import math import sys from pathlib import Path import numpy as np import soundfile as sf import torch from scipy.signal import resample_poly from transformers import WhisperFeatureExtractor REPO_ROOT = Path(__file__).resolve().parents[2] QWEN_ASR_CORE = REPO_ROOT / "Qwen3-ASR" / "qwen_asr" / "core" if str(QWEN_ASR_CORE) not in sys.path: sys.path.insert(0, str(QWEN_ASR_CORE)) from transformers_backend.modeling_qwen3_asr import ( # noqa: E402 Qwen3ASRForConditionalGeneration, ) TORCH_DTYPES = { "float16": torch.float16, "bfloat16": torch.bfloat16, "float32": torch.float32, } def get_torch_dtype(dtype: str) -> torch.dtype: if dtype not in TORCH_DTYPES: raise ValueError(f"Unsupported dtype: {dtype}") return TORCH_DTYPES[dtype] def load_waveform(audio_path: str, target_sr: int = 16000) -> np.ndarray: audio, sr = sf.read(audio_path, dtype="float32", always_2d=False) audio = np.asarray(audio, dtype=np.float32) if audio.ndim == 2: audio = audio.mean(axis=-1) if sr != target_sr: divisor = math.gcd(int(sr), int(target_sr)) up = int(target_sr // divisor) down = int(sr // divisor) audio = resample_poly(audio, up=up, down=down).astype(np.float32) return audio def configure_feature_extractor_for_audio(feature_extractor: WhisperFeatureExtractor, waveform: np.ndarray) -> None: required_seconds = max(1, math.ceil(waveform.shape[0] / float(feature_extractor.sampling_rate))) if required_seconds <= feature_extractor.chunk_length: return feature_extractor.chunk_length = required_seconds feature_extractor.n_samples = int(required_seconds * feature_extractor.sampling_rate) feature_extractor.nb_max_frames = feature_extractor.n_samples // feature_extractor.hop_length def extract_mel_features(model_path: str, audio_path: str) -> tuple[np.ndarray, int]: feature_extractor = WhisperFeatureExtractor.from_pretrained(model_path) waveform = load_waveform(audio_path) configure_feature_extractor_for_audio(feature_extractor, waveform) outputs = feature_extractor( waveform, sampling_rate=16000, return_attention_mask=True, return_tensors="np", ) input_features = outputs["input_features"][0].astype(np.float32) feature_len = int(outputs["attention_mask"][0].sum()) return input_features, feature_len def split_mel_features(input_features: np.ndarray, feature_len: int, chunk_frames: int) -> list[tuple[np.ndarray, int]]: chunks = [] start = 0 while start < feature_len: cur_len = min(chunk_frames, feature_len - start) chunk = np.zeros((input_features.shape[0], chunk_frames), dtype=np.float32) chunk[:, :cur_len] = input_features[:, start : start + cur_len] chunks.append((chunk, cur_len)) start += cur_len return chunks def load_audio_encoder(model_path: str, dtype: str = "float32", device: str = "cpu") -> torch.nn.Module: model = Qwen3ASRForConditionalGeneration.from_pretrained( model_path, dtype=get_torch_dtype(dtype), ) model = model.to(device) model.eval() tower = model.thinker.audio_tower tower.config._attn_implementation = "eager" for layer in tower.layers: layer.self_attn.config._attn_implementation = "eager" tower.eval() return tower def get_chunk_output_length(length: torch.Tensor) -> torch.Tensor: length = length.to(torch.int64) length = torch.div(length + 1, 2, rounding_mode="floor") length = torch.div(length + 1, 2, rounding_mode="floor") length = torch.div(length + 1, 2, rounding_mode="floor") return length def get_chunk_output_length_value(length: int) -> int: value = int(length) value = (value + 1) // 2 value = (value + 1) // 2 value = (value + 1) // 2 return value class StaticChunkAudioEncoder(torch.nn.Module): def __init__(self, tower: torch.nn.Module, chunk_frames: int = 100): super().__init__() self.tower = tower self.chunk_frames = int(chunk_frames) self.max_aftercnn_len = int(get_chunk_output_length(torch.tensor(self.chunk_frames)).item()) def forward(self, input_features: torch.Tensor, feature_len: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: batch_size = input_features.shape[0] outputs = [] valid_lens = [] for batch_idx in range(batch_size): current_feature_len = feature_len.reshape(-1)[batch_idx].to(torch.int64) padded = input_features[batch_idx, :, : self.chunk_frames].unsqueeze(0).unsqueeze(0) padded_embed = torch.nn.functional.gelu(self.tower.conv2d1(padded)) padded_embed = torch.nn.functional.gelu(self.tower.conv2d2(padded_embed)) padded_embed = torch.nn.functional.gelu(self.tower.conv2d3(padded_embed)) batch, channels, freq, time_steps = padded_embed.size() padded_embed = self.tower.conv_out( padded_embed.permute(0, 3, 1, 2).contiguous().view(batch, time_steps, channels * freq) ) positional_embedding = self.tower.positional_embedding.positional_embedding[:time_steps, :] padded_embed = padded_embed + positional_embedding.unsqueeze(0).to(padded_embed.dtype) hidden_states = padded_embed.squeeze(0) valid_len = get_chunk_output_length(current_feature_len.unsqueeze(0))[0].to(torch.int32) valid_positions = torch.arange(time_steps, device=hidden_states.device, dtype=torch.int32) < valid_len allowed = valid_positions[:, None] & valid_positions[None, :] zeros = torch.zeros((1, 1, time_steps, time_steps), dtype=hidden_states.dtype, device=hidden_states.device) minus_inf = torch.full( (1, 1, time_steps, time_steps), torch.finfo(hidden_states.dtype).min, dtype=hidden_states.dtype, device=hidden_states.device, ) attention_mask = torch.where(allowed.unsqueeze(0).unsqueeze(0), zeros, minus_inf) cu_seqlens = torch.stack( ( torch.zeros((), dtype=torch.int32, device=hidden_states.device), valid_len, ) ) for encoder_layer in self.tower.layers: hidden_states = encoder_layer( hidden_states, cu_seqlens=cu_seqlens, attention_mask=attention_mask, )[0] hidden_states = self.tower.ln_post(hidden_states) hidden_states = self.tower.proj1(hidden_states) hidden_states = self.tower.act(hidden_states) hidden_states = self.tower.proj2(hidden_states) outputs.append(hidden_states.unsqueeze(0)) valid_lens.append(valid_len) return torch.cat(outputs, dim=0), torch.stack(valid_lens, dim=0) def run_chunked_torch( model_path: str, input_features: np.ndarray, feature_len: int, chunk_frames: int = 100, dtype: str = "float32", device: str = "cpu", ) -> np.ndarray: tower = load_audio_encoder(model_path=model_path, dtype=dtype, device=device) wrapper = StaticChunkAudioEncoder(tower=tower, chunk_frames=chunk_frames).to(device).eval() outputs = [] for chunk, chunk_len in split_mel_features(input_features, feature_len, chunk_frames): chunk_tensor = torch.from_numpy(chunk).unsqueeze(0).to(device=device, dtype=get_torch_dtype(dtype)) chunk_len_tensor = torch.tensor([chunk_len], dtype=torch.int32, device=device) with torch.no_grad(): features, valid_len = wrapper(chunk_tensor, chunk_len_tensor) outputs.append(features[0, : valid_len[0].item()].detach().cpu().numpy()) return np.concatenate(outputs, axis=0)