# Adapted for diffusers from multimodal-art-projection/YuE at commit ef1936f2ee39fe8de486a0f47a481c95f8d4da87. # Licensed under Apache-2.0; see LICENSE. from dataclasses import replace import torch from diffusers import ModelMixin from diffusers.configuration_utils import FrozenDict from diffusers.modular_pipelines import ( ComponentSpec, InputParam, LoopSequentialPipelineBlocks, ModularPipelineBlocks, OutputParam, SequentialPipelineBlocks, ) from diffusers.utils.dynamic_modules_utils import get_class_from_dynamic_module from transformers import PreTrainedTokenizer from .cuda_graph import GraphAR from .guidance import YuE2SemanticGuider from .nar import YuE2PrefixKVCache, solve_midpoint, song_chunks from .protocol import ( ABC_SAMPLING, CODEC_OFFSET, CONTEXT, SEMANTIC_SAMPLING, SongRequest, negative_prefix, resolve_sampling, token_prefixes, ) from .sampling import generate_tokens def callback_inputs(): return [ InputParam( "cancelled", type_hint=object, default=None, description="Optional callable returning whether to cancel." ), InputParam( "on_token", type_hint=object, default=None, description="Optional callback receiving stage and token ID." ), ] def check_cancelled(callback): if callback is not None and callback(): raise InterruptedError("Generation cancelled") class YuE2PrepareInputsStep(ModularPipelineBlocks): model_name = "yue2" @property def description(self): return "Validates the song request." @property def inputs(self): return [ InputParam("style", type_hint=str, required=True, description="Music style description."), InputParam("lyrics", type_hint=str, required=True, description="Lyrics with song section tags."), InputParam("cot", type_hint=str, default="full", description="Score mode: full, melody, or off."), InputParam( "seed", type_hint=int, default=831001, description="Request seed. Each stage reseeds from it, as the original release does.", ), InputParam("abc", type_hint=str, default=None, description="Optional supplied ABC score."), ] @property def intermediate_outputs(self): return [OutputParam("request", type_hint=SongRequest, description="Validated song request.")] def __call__(self, components, state): block_state = self.get_block_state(state) block_state.request = SongRequest( style=block_state.style, lyrics=block_state.lyrics, cot=block_state.cot, seed=block_state.seed, abc=block_state.abc, ) self.set_block_state(state, block_state) return components, state class YuE2PlanStep(ModularPipelineBlocks): model_name = "yue2" _requirements = {"tiktoken": ">=0.12.0"} @property def description(self): return "Plans the song as an ABC score (or uses the supplied one) and builds the semantic-stage prompt." @property def expected_components(self): return [ComponentSpec("transformer", ModelMixin), ComponentSpec("tokenizer", PreTrainedTokenizer)] @property def inputs(self): return [ InputParam("request", type_hint=SongRequest, required=True, description="Validated song request."), InputParam( "abc_sampling", type_hint=dict, default=None, description="Overrides for the ABC sampling defaults." ), InputParam("use_cuda_graph", type_hint=bool, default=False, description="Decode tokens with CUDA graphs."), ] + callback_inputs() @property def intermediate_outputs(self): return [ OutputParam("score", type_hint=str, description="Generated or supplied ABC score; None with cot='off'."), OutputParam("abc_ids", type_hint=list, description="Exact score token IDs."), OutputParam("prefix", type_hint=list, description="Semantic-stage prompt token IDs."), OutputParam("abc_truncated", type_hint=bool, description="Whether planning hit its token limit."), OutputParam("abc_timing", type_hint=dict, description="Planning timings."), ] @torch.inference_mode() def __call__(self, components, state): block_state = self.get_block_state(state) check_cancelled(block_state.cancelled) request = block_state.request score, abc_ids, timing, truncated = None, [], {}, False if request.cot != "off": if request.abc is not None: score = request.abc abc_ids = components.tokenizer.encode(score) else: abc_ids, timing, truncated = generate_tokens( components.transformer, token_prefixes(request, components.tokenizer), resolve_sampling(block_state.abc_sampling, ABC_SAMPLING), request.seed, "abc", components._execution_device, cancelled=block_state.cancelled, on_token=block_state.on_token, graph_decoder=GraphAR if block_state.use_cuda_graph else None, ) score = components.tokenizer.decode(abc_ids) block_state.score = score block_state.abc_ids = abc_ids block_state.prefix = token_prefixes(request, components.tokenizer, abc_ids) block_state.abc_truncated = truncated block_state.abc_timing = timing self.set_block_state(state, block_state) return components, state class YuE2SemanticStep(ModularPipelineBlocks): model_name = "yue2" _requirements = {"tiktoken": ">=0.12.0"} @property def description(self): return "Generates the song's codec (semantic) tokens, with guidance from the `guider` component." @property def expected_components(self): return [ ComponentSpec("transformer", ModelMixin), ComponentSpec("tokenizer", PreTrainedTokenizer), ComponentSpec( "guider", YuE2SemanticGuider, config=FrozenDict({"scale": None}), default_creation_method="from_config" ), ] @property def inputs(self): return [ InputParam("request", type_hint=SongRequest, required=True, description="Validated song request."), InputParam("abc_ids", type_hint=list, required=True, description="Exact score token IDs."), InputParam("prefix", type_hint=list, required=True, description="Semantic-stage prompt token IDs."), InputParam( "semantic_sampling", type_hint=dict, default=None, description="Overrides for the semantic sampling defaults.", ), InputParam("use_cuda_graph", type_hint=bool, default=False, description="Decode tokens with CUDA graphs."), ] + callback_inputs() @property def intermediate_outputs(self): return [ OutputParam("semantic_tokens", type_hint=list, description="Codec token IDs, 25 per second of audio."), OutputParam("semantic_truncated", type_hint=bool, description="Whether generation hit its token limit."), OutputParam("semantic_timing", type_hint=dict, description="Semantic-stage timings."), ] @torch.inference_mode() def __call__(self, components, state): block_state = self.get_block_state(state) check_cancelled(block_state.cancelled) request = block_state.request if token_prefixes(request, components.tokenizer, block_state.abc_ids) != block_state.prefix: raise ValueError("The semantic prompt disagrees with the request and score IDs") guider = components.guider negative = None if guider.scale_for(request.cot) != 1: negative = negative_prefix(request, components.tokenizer, block_state.abc_ids) tokens, timing, truncated = generate_tokens( components.transformer, block_state.prefix, resolve_sampling(block_state.semantic_sampling, SEMANTIC_SAMPLING), request.seed, "semantic", components._execution_device, negative=negative, combine_logits=lambda conditional, unconditional: guider.combine(conditional, unconditional, request.cot), legacy_off=request.cot == "off", cancelled=block_state.cancelled, on_token=block_state.on_token, graph_decoder=GraphAR if block_state.use_cuda_graph else None, ) block_state.semantic_tokens = [int(t) - CODEC_OFFSET for t in tokens] block_state.semantic_truncated = truncated block_state.semantic_timing = timing self.set_block_state(state, block_state) return components, state class YuE2PrepareChunksStep(ModularPipelineBlocks): model_name = "yue2" @property def description(self): return "Splits the song into acoustic chunks and draws their seeded noise." @property def inputs(self): return [ InputParam("request", type_hint=SongRequest, required=True, description="Validated song request."), InputParam("prefix", type_hint=list, required=True, description="Semantic-stage prompt token IDs."), InputParam("semantic_tokens", type_hint=list, required=True, description="Codec token IDs."), InputParam( "acoustic_context", type_hint=int, default=CONTEXT, description="Context limit that sets the acoustic chunk size.", ), ] @property def intermediate_outputs(self): return [OutputParam("chunks", type_hint=list, description="Acoustic chunk tokens and CPU FP32 noise.")] def __call__(self, components, state): block_state = self.get_block_state(state) block_state.chunks = song_chunks( block_state.prefix, block_state.semantic_tokens, block_state.request.seed, block_state.acoustic_context, ) self.set_block_state(state, block_state) return components, state class YuE2ChunkConditionStep(ModularPipelineBlocks): model_name = "yue2" @property def description(self): return "Runs the chunk's tokens through the token stream once and keeps their keys and values." @property def expected_components(self): return [ComponentSpec("transformer", ModelMixin)] @property def inputs(self): return [ InputParam("chunks", type_hint=list, required=True, description="Prepared acoustic chunks."), InputParam("cancelled", type_hint=object, default=None, description="Cancellation callable."), ] @torch.inference_mode() def __call__(self, components, block_state, k): check_cancelled(block_state.cancelled) block_state.chunk = block_state.chunks[k] block_state.kv_cache = YuE2PrefixKVCache() ids = torch.tensor([block_state.chunk.ar_tokens], device=components._execution_device) components.transformer(ids, kv_cache=block_state.kv_cache, logits_to_keep=1) return components, block_state class YuE2ChunkSynthesizeStep(ModularPipelineBlocks): model_name = "yue2" @property def description(self): return "Solves the chunk's flow-matching ODE with the midpoint method." @property def expected_components(self): return [ComponentSpec("transformer", ModelMixin)] @property def inputs(self): return [ InputParam("chunks", type_hint=list, required=True, description="Prepared acoustic chunks."), InputParam("ode_steps", type_hint=int, default=32, description="Midpoint steps per chunk."), InputParam("cancelled", type_hint=object, default=None, description="Cancellation callable."), InputParam( "on_progress", type_hint=object, default=None, description="Callback(stage, completed, total)." ), ] @torch.inference_mode() def __call__(self, components, block_state, k): report = None if block_state.on_progress is not None: def report(done, total): block_state.on_progress("synthesis", k * total + done, len(block_state.chunks) * total) block_state.chunk_latents = solve_midpoint( components.transformer, block_state.kv_cache, block_state.chunk.noise, components._execution_device, block_state.ode_steps, block_state.cancelled, report, ) return components, block_state class YuE2CollectChunkStep(ModularPipelineBlocks): model_name = "yue2" @property def description(self): return "Appends the chunk's latents." def __call__(self, components, block_state, k): block_state.latent_chunks.append(block_state.chunk_latents) return components, block_state class YuE2AcousticChunkLoop(LoopSequentialPipelineBlocks): model_name = "yue2" block_classes = [YuE2ChunkConditionStep, YuE2ChunkSynthesizeStep, YuE2CollectChunkStep] block_names = ["condition", "synthesize", "collect"] @property def description(self): return "Synthesizes acoustic latents chunk by chunk." @property def loop_inputs(self): return [InputParam("chunks", type_hint=list, required=True, description="Prepared acoustic chunks.")] @property def loop_intermediate_outputs(self): return [OutputParam("latents", type_hint=torch.Tensor, description="Acoustic latents [frames, 64], CPU FP32.")] @torch.inference_mode() def __call__(self, components, state): block_state = self.get_block_state(state) if not block_state.chunks: raise ValueError("At least one acoustic chunk is required") block_state.latent_chunks = [] for k in range(len(block_state.chunks)): try: components, block_state = self.loop_step(components, block_state, k=k) finally: block_state.kv_cache = None block_state.latents = torch.cat(block_state.latent_chunks, dim=0) self.set_block_state(state, block_state) return components, state class YuE2SynthesizeStep(SequentialPipelineBlocks): model_name = "yue2" block_classes = [YuE2PrepareChunksStep, YuE2AcousticChunkLoop] block_names = ["prepare_chunks", "chunk_loop"] @property def description(self): return "Turns codec tokens into acoustic latents." class YuE2DecodeStep(ModularPipelineBlocks): model_name = "yue2" @property def description(self): return ( "Decodes latents to 48 kHz stereo audio in overlapping tiles. The VAE runs in FP32 with TF32 and cuDNN " "autotuning disabled, as in the original release." ) @property def expected_components(self): return [ComponentSpec("vae", ModelMixin)] @property def inputs(self): return [ InputParam( "latents", type_hint=torch.Tensor, required=True, description="Acoustic latents, [frames, 64] or [1, 64, frames].", ), InputParam("vae_tile_frames", type_hint=int, default=1024, description="Latent frames decoded per tile."), InputParam( "vae_tile_overlap_frames", type_hint=int, default=16, description="Context frames decoded on each side of a tile and cropped away.", ), InputParam("cancelled", type_hint=object, default=None, description="Cancellation callable."), InputParam( "on_progress", type_hint=object, default=None, description="Callback(stage, completed, total)." ), ] @property def intermediate_outputs(self): return [ OutputParam("audios", type_hint=torch.Tensor, description="Stereo audio [1, 2, samples], CPU FP32."), OutputParam("sample_rate", type_hint=int, description="Sample rate in Hz."), ] @torch.inference_mode() def __call__(self, components, state): block_state = self.get_block_state(state) check_cancelled(block_state.cancelled) vae = components.vae device = components._execution_device latents = torch.as_tensor(block_state.latents, dtype=torch.float32) if latents.ndim == 2: latents = latents.T.unsqueeze(0) frames = latents.shape[-1] tile, overlap = block_state.vae_tile_frames, block_state.vae_tile_overlap_frames if tile < 1 or overlap < 0: raise ValueError("vae_tile_frames must be positive and vae_tile_overlap_frames nonnegative") pieces = [] starts = range(0, frames, tile) with torch.backends.cudnn.flags(enabled=True, benchmark=False, deterministic=True, allow_tf32=False): for index, start in enumerate(starts): check_cancelled(block_state.cancelled) end = min(frames, start + tile) left, right = max(0, start - overlap), min(frames, end + overlap) decoded = vae.decode(latents[..., left:right].to(device=device, dtype=torch.float32)).sample crop = (start - left) * vae.hop_length if end < frames: decoded = decoded[..., crop : crop + (end - start) * vae.hop_length] else: decoded = decoded[..., crop:] pieces.append(decoded.cpu()) if block_state.on_progress is not None: block_state.on_progress("decode", index + 1, len(starts)) audio = torch.cat(pieces, dim=-1) if not torch.isfinite(audio).all(): raise FloatingPointError("Decoded audio is non-finite") block_state.audios = audio.clamp(-1, 1) block_state.sample_rate = vae.config.sampling_rate self.set_block_state(state, block_state) return components, state class YuE2Blocks(SequentialPipelineBlocks): model_name = "yue2" block_names = ["prepare", "plan", "semantic", "synthesize", "decode"] block_classes = [YuE2PrepareInputsStep, YuE2PlanStep, YuE2SemanticStep, YuE2SynthesizeStep, YuE2DecodeStep] def __init__(self, components_repo=None, components_revision=None, trust_components_code=False): super().__init__() self.components_repo = components_repo self.components_revision = components_revision self.component_types = {} if components_repo is not None: if not trust_components_code: raise ValueError("Set trust_components_code=True to load code from the model repository") for name, class_name in ( ("transformer", "YuE2TransformerModel"), ("vae", "YuE2VAE"), ("tokenizer", "YuE2Tokenizer"), ): self.component_types[name] = get_class_from_dynamic_module( components_repo, module_file=f"{name}.py", class_name=class_name, revision=components_revision, trust_remote_code=True, ) @property def expected_components(self): specs = super().expected_components if self.components_repo is None: return specs return [ replace( spec, type_hint=self.component_types[spec.name], pretrained_model_name_or_path=str(self.components_repo), subfolder=spec.name, revision=self.components_revision, ) if spec.default_creation_method == "from_pretrained" else spec for spec in specs ] @property def description(self): return "YuE2 score planning, semantic generation, midpoint synthesis, and stereo decoding."