"""Shared audio primitives for LTX-2.5 task tabs. This module owns raw-audio decoding/preprocessing and the narrow Diffusers adaptation required to treat an audio modality as frozen conditioning. Task semantics and evidence remain in the individual tab module. """ from __future__ import annotations from contextlib import contextmanager from enum import StrEnum from pathlib import Path import numpy as np import torch from diffusers import FlowMatchEulerDiscreteScheduler _INT_FORMAT_MAX = { "u8": 255.0, "u8p": 255.0, "s16": 32768.0, "s16p": 32768.0, "s32": 2147483648.0, "s32p": 2147483648.0, } class AudioAuthority(StrEnum): """Which waveform is authoritative for a task's final audio output. This is deliberately a small vocabulary, not a task framework. Individual task modules continue to own conditioning, diffusion and mux semantics. """ MODEL_GENERATED_JOINT = "model_generated_joint" EXTERNAL_FIXED_CONDITIONING_PASSTHROUGH = "external_fixed_conditioning_passthrough" SOURCE_VIDEO_PASSTHROUGH = "source_video_passthrough" EXTERNAL_REPLACEMENT = "external_replacement" GENERATED_AUDIO_OUTPUT = "generated_audio_output" _AUDIO_AUTHORITY_MEANINGS = { AudioAuthority.MODEL_GENERATED_JOINT: "model-generated joint A/V audio is authoritative", AudioAuthority.EXTERNAL_FIXED_CONDITIONING_PASSTHROUGH: ( "external audio conditions generation while its original decoded waveform remains authoritative" ), AudioAuthority.SOURCE_VIDEO_PASSTHROUGH: "source-video audio is preserved unchanged as final output audio", AudioAuthority.EXTERNAL_REPLACEMENT: "an explicit external replacement waveform is authoritative", AudioAuthority.GENERATED_AUDIO_OUTPUT: "the task's generated audio waveform is authoritative", } def audio_authority_metadata(authority: AudioAuthority | str) -> dict[str, str]: """Return a stable diagnostic record for one authority vocabulary value.""" resolved = AudioAuthority(authority) return {"kind": resolved.value, "meaning": _AUDIO_AUTHORITY_MEANINGS[resolved]} def _audio_frame_to_float(frame) -> np.ndarray: """Convert a PyAV AudioFrame to float32 [channels, samples] in [-1, 1].""" arr = frame.to_ndarray().astype(np.float32) maximum = _INT_FORMAT_MAX.get(frame.format.name) if maximum is not None: arr = arr / maximum if not frame.format.is_planar: channels = len(frame.layout.channels) arr = arr.reshape(-1, channels).T return arr def decode_audio_file(path: str | Path, *, max_duration: float) -> tuple[np.ndarray, int, dict]: """Decode the beginning of an audio/video file using PyAV. Returns stereo float32 waveform [2, samples], source sample rate and metadata. The first A2V probe intentionally requires enough source audio for the whole requested clip instead of silently padding missing content. """ import av source_path = Path(path) container = av.open(str(source_path)) try: stream = next((item for item in container.streams if item.type == "audio"), None) if stream is None: raise ValueError("The uploaded file has no audio stream.") sample_rate = int(stream.rate) samples = [] decoded_frames = 0 for frame in container.decode(stream): samples.append(_audio_frame_to_float(frame)) decoded_frames += 1 if sum(item.shape[-1] for item in samples) >= int(round(float(max_duration) * sample_rate)): break finally: container.close() if not samples: raise ValueError("The uploaded audio decoded zero samples.") waveform = np.concatenate(samples, axis=-1).astype(np.float32, copy=False) required_samples = int(round(float(max_duration) * sample_rate)) if waveform.shape[-1] < required_samples: actual_seconds = waveform.shape[-1] / float(sample_rate) raise ValueError( f"Input audio is too short for this probe: {actual_seconds:.3f}s available, " f"{float(max_duration):.3f}s required." ) waveform = waveform[..., :required_samples] source_channels = int(waveform.shape[0]) if source_channels == 1: waveform = np.repeat(waveform, 2, axis=0) channel_policy = "mono duplicated to stereo" elif source_channels >= 2: waveform = waveform[:2] channel_policy = "first two channels preserved" if source_channels > 2 else "stereo preserved" else: raise ValueError(f"Unsupported decoded channel count: {source_channels}") waveform = np.clip(waveform, -1.0, 1.0).astype(np.float32, copy=False) return waveform, sample_rate, { "source_channels": source_channels, "condition_channels": 2, "channel_policy": channel_policy, "decoded_frames": decoded_frames, "source_sample_rate": sample_rate, "trimmed_samples": int(waveform.shape[-1]), "trimmed_duration_seconds": waveform.shape[-1] / float(sample_rate), } def waveform_to_log_mel( waveform: np.ndarray, source_sample_rate: int, *, target_sample_rate: int, mel_bins: int, mel_hop_length: int, n_fft: int = 1024, ) -> tuple[np.ndarray, dict]: """Match the official LTX AudioProcessor frontend on CPU. Uses TorchAudio only for resampling/MelSpectrogram; no TorchCodec I/O path is used because input decoding is handled by PyAV. """ import torchaudio tensor = torch.from_numpy(np.asarray(waveform, dtype=np.float32)).unsqueeze(0) if int(source_sample_rate) != int(target_sample_rate): tensor = torchaudio.functional.resample(tensor, int(source_sample_rate), int(target_sample_rate)) transform = torchaudio.transforms.MelSpectrogram( sample_rate=int(target_sample_rate), n_fft=int(n_fft), win_length=int(n_fft), hop_length=int(mel_hop_length), f_min=0.0, f_max=float(target_sample_rate) / 2.0, n_mels=int(mel_bins), window_fn=torch.hann_window, center=True, pad_mode="reflect", power=1.0, mel_scale="slaney", norm="slaney", ) mel = transform(tensor) mel = torch.log(torch.clamp(mel, min=1e-5)) # Official ltx-core AudioProcessor returns [batch, channels, time, n_mels]. mel = mel.permute(0, 1, 3, 2).contiguous().float().cpu() return mel.numpy(), { "frontend": "official-compatible log-mel", "target_sample_rate": int(target_sample_rate), "mel_bins": int(mel_bins), "mel_hop_length": int(mel_hop_length), "n_fft": int(n_fft), "mel_shape": [int(value) for value in mel.shape], } def expected_audio_latent_frames(*, num_frames: int, frame_rate: float, sample_rate: int, hop_length: int, compression: int) -> int: duration_seconds = float(num_frames) / float(frame_rate) per_second = float(sample_rate) / float(hop_length) / float(compression) return int(round(duration_seconds * per_second)) def encode_clean_audio_latents(pipe, mel: np.ndarray, *, expected_frames: int) -> torch.Tensor: """Encode precomputed log-mel input with Diffusers' already-resident audio VAE.""" mel_tensor = torch.from_numpy(np.asarray(mel, dtype=np.float32)).to( device=pipe._execution_device, dtype=pipe.audio_vae.dtype ) with torch.inference_mode(): posterior = pipe.audio_vae.encode(mel_tensor).latent_dist latents = posterior.mode() if latents.shape[2] < int(expected_frames): raise RuntimeError( f"Encoded audio latent is shorter than the requested video: {latents.shape[2]} < {int(expected_frames)} frames." ) return latents[:, :, : int(expected_frames), :].contiguous() class _FrozenAudioScheduler(FlowMatchEulerDiscreteScheduler): """Audio-only deepcopy of the video scheduler whose step leaves the sample unchanged.""" def step(self, model_output, timestep, sample, *args, return_dict: bool = True, **kwargs): if return_dict: # The LTX2 pipeline always requests return_dict=False. Keep failure explicit # rather than constructing a scheduler-specific output type here. raise RuntimeError("Frozen audio scheduler expects return_dict=False.") return (sample,) class FrozenAudioCopyScheduler(FlowMatchEulerDiscreteScheduler): """Normal video scheduler that deep-copies into a frozen audio scheduler. LTX2Pipeline currently creates its audio scheduler with copy.deepcopy(self.scheduler). This preserves normal video scheduler behaviour while making only that private audio copy immutable for A2V. """ def __deepcopy__(self, memo): frozen = _FrozenAudioScheduler.from_config(self.config) memo[id(self)] = frozen return frozen def frozen_audio_scheduler_from_config(config, **kwargs) -> FrozenAudioCopyScheduler: return FrozenAudioCopyScheduler.from_config(config, **kwargs) @contextmanager def frozen_audio_conditioning(pipe, *, scheduler: FrozenAudioCopyScheduler): """Give one Diffusers LTX2 call the native A2V frozen-audio contract. Diffusers exposes the right multimodal model pieces, but its ordinary I2V pipeline denoises both modalities. Native A2V instead keeps the input audio latent fixed while video is denoised. This task-scoped adapter therefore: - normalizes/packs supplied audio latents without adding noise; - forces audio timestep/sigma to zero; - keeps video scale/shift on the video timestep while overriding only the A2V/V2A cross-attention gate inputs to the opposite modality sigma; - replaces only the pipeline's private audio scheduler copy with an identity-step scheduler. All hooks and monkeypatches are removed in ``finally``. The yielded mutable evidence dict is filled with hook-count/scale metadata and can be recorded by the task module after the context exits. """ original_scheduler = pipe.scheduler original_prepare_audio_latents = pipe.prepare_audio_latents transformer = pipe.transformer video_gate = getattr(transformer, "av_cross_attn_video_a2v_gate", None) audio_gate = getattr(transformer, "av_cross_attn_audio_v2a_gate", None) if video_gate is None or audio_gate is None: raise RuntimeError("Current LTX2 transformer does not expose the expected A2V/V2A gate modules.") config = transformer.config timestep_multiplier = float(getattr(config, "timestep_scale_multiplier", 1.0) or 1.0) cross_multiplier = float(getattr(config, "cross_attn_timestep_scale_multiplier", timestep_multiplier)) gate_scale_ratio = cross_multiplier / timestep_multiplier state = {"video_sigma": None} audio_self_attn_modules = [ block.audio_attn1 for block in getattr(transformer, "transformer_blocks", []) if getattr(block, "audio_attn1", None) is not None ] def _hook_modules(): return [transformer, video_gate, audio_gate, *audio_self_attn_modules] def _hook_count() -> int: return sum(len(getattr(module, "_forward_pre_hooks", {})) for module in _hook_modules()) baseline_hook_count = _hook_count() evidence = { "baseline_forward_pre_hook_count": baseline_hook_count, "cross_attn_gate_scale_ratio": gate_scale_ratio, "video_gate_input": "frozen audio sigma = 0", "audio_gate_input": "current video sigma", "scale_shift_policy": "per-modality timestep; use_cross_timestep=False", "cfg_audio_context_policy": "positive audio context reused for the video-CFG unconditioned half", "stg_audio_policy": "audio self-attention remains unperturbed; STG perturbs video only", "cfg_audio_context_replacements": 0, "audio_self_attention_unperturb_hooks": len(audio_self_attn_modules), } def _prepare_audio_latents_without_noise(*args, **kwargs): kwargs["noise_scale"] = 0.0 return original_prepare_audio_latents(*args, **kwargs) def _freeze_audio_timestep(module, args, kwargs): video_timestep = kwargs.get("timestep") video_sigma = kwargs.get("sigma") audio_timestep = kwargs.get("audio_timestep") audio_sigma = kwargs.get("audio_sigma") state["video_sigma"] = video_sigma if video_sigma is not None else video_timestep # Diffusers I2V uses a per-token video timestep (B, T_video), while # audio_timestep/audio_sigma are per-batch (B,). Frozen A2V must zero # the existing audio tensors without inheriting the video-token shape. # Native LTX keeps Modality.sigma per-batch and derives per-token # timesteps separately from the modality denoise mask. if isinstance(audio_timestep, torch.Tensor): kwargs["audio_timestep"] = torch.zeros_like(audio_timestep) elif isinstance(video_sigma, torch.Tensor): kwargs["audio_timestep"] = torch.zeros_like(video_sigma) elif isinstance(video_timestep, torch.Tensor): fallback = video_timestep if fallback.ndim > 1: fallback = fallback.reshape(fallback.shape[0], -1)[:, 0] kwargs["audio_timestep"] = torch.zeros_like(fallback) if isinstance(audio_sigma, torch.Tensor): kwargs["audio_sigma"] = torch.zeros_like(audio_sigma) elif isinstance(video_sigma, torch.Tensor): kwargs["audio_sigma"] = torch.zeros_like(video_sigma) elif isinstance(kwargs.get("audio_timestep"), torch.Tensor): kwargs["audio_sigma"] = torch.zeros_like(kwargs["audio_timestep"]) # Native A2V uses a positive-only audio guider. When Diffusers batches # [video-uncond, video-cond] for CFG, reuse the positive audio prompt # context in both halves so only the video text context is unconditioned. audio_context = kwargs.get("audio_encoder_hidden_states") if ( isinstance(audio_context, torch.Tensor) and audio_context.ndim >= 1 and audio_context.shape[0] == 2 and isinstance(video_timestep, torch.Tensor) and video_timestep.shape[0] == 2 ): positive_audio_context = audio_context[1:2] kwargs["audio_encoder_hidden_states"] = torch.cat( [positive_audio_context, positive_audio_context], dim=0 ) audio_mask = kwargs.get("audio_encoder_attention_mask") if isinstance(audio_mask, torch.Tensor) and audio_mask.shape[0] == 2: positive_audio_mask = audio_mask[1:2] kwargs["audio_encoder_attention_mask"] = torch.cat( [positive_audio_mask, positive_audio_mask], dim=0 ) evidence["cfg_audio_context_replacements"] += 1 return args, kwargs def _replace_first_arg(args, replacement): if not args: raise RuntimeError("LTX2 cross-attention gate hook received no positional timestep input.") return (replacement,) + tuple(args[1:]) def _zero_video_gate(module, args): reference = args[0] return _replace_first_arg(args, torch.zeros_like(reference)) def _video_sigma_for_audio_gate(module, args): reference = args[0] sigma = state.get("video_sigma") if sigma is None: raise RuntimeError("Video sigma was not captured before the V2A gate was evaluated.") sigma = sigma.to(device=reference.device, dtype=reference.dtype) * gate_scale_ratio if sigma.shape != reference.shape: try: sigma = torch.broadcast_to(sigma, reference.shape) except RuntimeError as exc: raise RuntimeError( f"Cannot broadcast video sigma shape {tuple(sigma.shape)} to V2A gate input {tuple(reference.shape)}." ) from exc return _replace_first_arg(args, sigma) def _keep_audio_self_attention_unperturbed(module, args, kwargs): if "perturbation_mask" in kwargs: kwargs["perturbation_mask"] = None kwargs["all_perturbed"] = False return args, kwargs transformer_hook = transformer.register_forward_pre_hook(_freeze_audio_timestep, with_kwargs=True) video_gate_hook = video_gate.register_forward_pre_hook(_zero_video_gate) audio_gate_hook = audio_gate.register_forward_pre_hook(_video_sigma_for_audio_gate) audio_self_attn_hooks = [ module.register_forward_pre_hook(_keep_audio_self_attention_unperturbed, with_kwargs=True) for module in audio_self_attn_modules ] pipe.scheduler = scheduler pipe.prepare_audio_latents = _prepare_audio_latents_without_noise evidence["active_forward_pre_hook_count"] = _hook_count() try: yield evidence finally: pipe.prepare_audio_latents = original_prepare_audio_latents pipe.scheduler = original_scheduler for hook in reversed(audio_self_attn_hooks): hook.remove() audio_gate_hook.remove() video_gate_hook.remove() transformer_hook.remove() restored_hook_count = _hook_count() evidence["restored_forward_pre_hook_count"] = restored_hook_count evidence["hooks_restored"] = restored_hook_count == baseline_hook_count if restored_hook_count != baseline_hook_count: raise RuntimeError( "Frozen-audio transformer hooks were not fully restored: " f"baseline={baseline_hook_count}, restored={restored_hook_count}." )