nar.py

# Adapted for diffusers from multimodal-art-projection/YuE at commit ef1936f2ee39fe8de486a0f47a481c95f8d4da87.
# Licensed under Apache-2.0; see LICENSE.
from __future__ import annotations

from collections.abc import Callable
from dataclasses import dataclass
from numbers import Integral

import torch

from .protocol import CODEC_OFFSET, CODEC_SIZE, CONTEXT, MUSIC_END, chunk_ranges


@dataclass
class Chunk:
    ar_tokens: list[int]
    noise: torch.Tensor


class YuE2PrefixKVCache:
    """Holds a chunk's token-prefix keys and values; later acoustic calls attend to the prefix without extending it."""

    def __init__(self):
        self.keys, self.values = [], []

    def get_seq_length(self):
        return self.keys[0].shape[1] if self.keys else 0

    def update(self, key, value, layer_idx):
        if layer_idx == len(self.keys):
            self.keys.append(key)
            self.values.append(value)
            return key, value
        return torch.cat((self.keys[layer_idx], key), dim=1), torch.cat((self.values[layer_idx], value), dim=1)


def _integers(values, name):
    result = list(values)
    if not result or any(isinstance(v, bool) or not isinstance(v, Integral) for v in result):
        raise ValueError(f"{name} must be a nonempty sequence of integer token IDs")
    return [int(v) for v in result]


def song_chunks(prefix, codec, seed, context=CONTEXT):
    """Draw the whole song's CPU FP32 noise once, then split it at the release's chunk boundaries."""
    prefix = _integers(prefix, "prefix")
    codec = _integers(codec, "codec")
    if min(prefix) < 0 or min(codec) < 0 or max(codec) >= CODEC_SIZE:
        raise ValueError("Token IDs are outside their allowed vocabulary")
    if isinstance(context, bool) or not isinstance(context, Integral) or not 1 <= context <= CONTEXT:
        raise ValueError(f"context must be an integer in 1..{CONTEXT}")
    ranges = chunk_ranges(len(codec), len(prefix), int(context))
    generator = torch.Generator(device="cpu").manual_seed(int(seed))
    noise = torch.randn((len(codec), 64), dtype=torch.float32, device="cpu", generator=generator)
    return [
        Chunk(prefix + [value + CODEC_OFFSET for value in codec[a:b]] + [MUSIC_END], noise[a:b]) for a, b in ranges
    ]


@torch.inference_mode()
def solve_midpoint(
    transformer,
    kv_cache,
    noise,
    device,
    steps=32,
    cancelled: Callable[[], bool] | None = None,
    on_progress: Callable[[int, int], None] | None = None,
):
    """Integrate the flow from t=1 (noise) to t=0 with the midpoint method; returns CPU FP32 [frames, 64] latents."""
    if isinstance(steps, bool) or not isinstance(steps, Integral) or steps < 1:
        raise ValueError("steps must be a positive integer")
    if not torch.isfinite(noise).all():
        raise ValueError("Acoustic noise contains non-finite values")

    def velocity(state, t):
        # The model takes time in logit space; logit(1) is clamped to 20.
        raw = torch.logit(torch.tensor(t, dtype=torch.float64, device="cpu")).clamp(-20, 20).item()
        return transformer(latents=state[None], timestep=raw, kv_cache=kv_cache).sample[0]

    state = noise.to(device=device, dtype=transformer.dtype)
    dt = 1.0 / steps
    for step in range(steps):
        if cancelled is not None and cancelled():
            raise InterruptedError("Cancelled during acoustic flow matching")
        t = 1.0 - step * dt
        mid = state - velocity(state, t) * (dt / 2)
        if cancelled is not None and cancelled():
            raise InterruptedError("Cancelled during acoustic flow matching")
        state = state - velocity(mid, t - dt / 2) * dt
        if on_progress is not None:
            on_progress(step + 1, int(steps))
    result = state.float().cpu()
    if not torch.isfinite(result).all():
        raise FloatingPointError("Acoustic flow matching produced non-finite latents")
    return result