Quantumbraid's picture
download
raw
6.21 kB
from typing import List, Optional
import torch
import time
import os
from .model import (
SAMPLE_RATE,
AceStep15,
OobleckVAE,
TextEncoder,
get_silence_latent,
compute_timesteps,
)
from diffusers.utils.torch_utils import randn_tensor
from transformers import AutoTokenizer
SFT_PROMPT = """# Instruction
{instruction}
# Caption
{caption}
# Metas
{metas}<|endoftext|>
"""
class AceStep15Pipeline:
SAMPLE_RATE = 48000
LATENT_RATE = 25 # 48000 / 1920
SFT_PROMPT = SFT_PROMPT
def __init__(self, transformer, vae, text_encoder, tokenizer, scheduler):
self.transformer: AceStep15 = transformer
self.vae: OobleckVAE = vae
self.text_encoder: TextEncoder = text_encoder
self.tokenizer: AutoTokenizer = tokenizer
self.scheduler = scheduler
self.do_tiled_decoding = False
def to(self, *args, **kwargs):
self.transformer.to(*args, **kwargs)
self.vae.to(*args, **kwargs)
self.text_encoder.to(*args, **kwargs)
def get_text_embedings(
self, prompt, lyrics, bpm, key, time_sig, duration, language
):
metas = f"- bpm: {bpm}\n- timesignature: {time_sig}\n- keyscale: {key}\n- duration: {int(duration)} seconds\n"
caption = self.SFT_PROMPT.format(
instruction="Fill the audio semantic mask based on the given conditions:",
caption=prompt,
metas=metas,
)
lyrics_text = f"# Languages\n{language}\n\n# Lyric\n{lyrics}<|endoftext|>"
cap_tok = self.tokenizer(
caption, truncation=True, max_length=256, return_tensors="pt"
)
lyr_tok = self.tokenizer(
lyrics_text, truncation=True, max_length=2048, return_tensors="pt"
)
text_embeddings = self.text_encoder.encode_text(
cap_tok.input_ids.to(self.text_encoder.device)
).to(self.transformer.dtype)
text_mask = cap_tok.attention_mask.to(self.text_encoder.device).bool()
lyric_embeddings = self.text_encoder.encode_lyrics(
lyr_tok.input_ids.to(self.text_encoder.device)
).to(self.transformer.dtype)
lyric_mask = lyr_tok.attention_mask.to(self.text_encoder.device).bool()
return text_embeddings, text_mask, lyric_embeddings, lyric_mask
def __call__(
self,
prompt="",
lyrics="",
encoder_embeddings: Optional[List[torch.Tensor]] = None,
encoder_mask: Optional[List[torch.Tensor]] = None,
# uses a null conditional for unconditional if not provided, which is what we want for CFG
num_inference_steps=50,
duration=30.0,
generator: torch.Generator = None,
bpm="N/A",
key="N/A",
time_sig="N/A",
language="en",
guidance_scale=1.0,
step_callback=None, # step_callback(i, num_steps, latents) after each update
):
t_sched = compute_timesteps(num_inference_steps, 3.0)
latent_len = int(duration * self.LATENT_RATE)
device = self.transformer.device
dtype = self.transformer.dtype
# Text encoding
if encoder_embeddings is not None and encoder_mask is not None:
enc_h = encoder_embeddings
enc_m = encoder_mask
sil = get_silence_latent(latent_len, device, dtype) # [1, 64, T]
src = sil.transpose(1, 2) # [1, T, 64]
chunk_masks = torch.ones_like(src)
ctx = torch.cat([src, chunk_masks.to(src.dtype)], dim=-1)
else:
text_h, text_m, lyric_h, lyric_m = self.get_text_embedings(
prompt, lyrics, bpm, key, time_sig, duration, language
)
# Silence as source latent [1, 64, T] -> [1, T, 64] for DiT
sil = get_silence_latent(latent_len, device, dtype) # [1, 64, T]
src = sil.transpose(1, 2) # [1, T, 64]
chunk_masks = torch.ones_like(src)
# Reference audio (silence)
ref = sil[:, :, :750].transpose(1, 2) # [1, 750, 64]
ref_order = torch.zeros(1, device=device, dtype=torch.long)
# Prepare conditions (conditional)
enc_h, enc_m, ctx = self.transformer.prepare_condition(
text_h, text_m, lyric_h, lyric_m, ref, ref_order, src, chunk_masks
)
# Prepare unconditional conditions for CFG
use_cfg = guidance_scale > 1.0
enc_h_uncond = None
if use_cfg:
enc_h_uncond = self.transformer.null_condition_emb.expand_as(enc_h)
# Noise
if generator is None:
generator = torch.Generator(device=device)
noise_ch = ctx.shape[-1] // 2
xt = randn_tensor(
(1, latent_len, noise_ch), generator=generator, device=device, dtype=dtype
)
# xt = torch.randn(1, latent_len, noise_ch, generator=generator, device=device, dtype=dtype)
# Diffusion
t_sched_t = torch.tensor(t_sched, device=device, dtype=dtype)
attn = torch.ones(1, latent_len, device=device, dtype=dtype)
for i in range(len(t_sched_t)):
tv = t_sched_t[i].item()
tt = torch.full((1,), tv, device=device, dtype=dtype)
vt_cond = self.transformer.decoder(xt, tt, tt, attn, enc_h, enc_m, ctx)
if use_cfg:
vt_uncond = self.transformer.decoder(
xt, tt, tt, attn, enc_h_uncond, enc_m, ctx
)
vt = vt_uncond + guidance_scale * (vt_cond - vt_uncond)
else:
vt = vt_cond
if step_callback is not None:
# x0 estimate at this step (flow matching: x_t - t * v)
step_callback(i, len(t_sched_t), xt - vt * tv)
if i == len(t_sched_t) - 1:
xt = xt - vt * tv
else:
xt = xt - vt * (tv - t_sched_t[i + 1].item())
# VAE decode
if self.do_tiled_decoding:
wav = self.vae.tiled_decode(xt.transpose(1, 2)) # [1, 2, samples]
else:
wav = self.vae.decode(xt.transpose(1, 2)) # [1, 2, samples]
wav = wav[0, :, : int(duration * SAMPLE_RATE)]
return wav

Xet Storage Details

Size:
6.21 kB
·
Xet hash:
687d587e131c75a4e85596ddba1f338133135ec7ff93f66ed8e530e1e1b06142

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.