Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / diffusers /pr_14159 /en /_app /immutable /nodes /164.CEGLSS1w.js

HuggingFaceDocBuilder

16 days ago

download

raw

14.4 kB

import"../chunks/DsnmJJEf.js";import{i as J,h as j,H as o,a as T,D as u,E as v,s as W}from"../chunks/BLbO27j4.js";import{p as k,o as B,s as e,f as Z,a as h,b as I,c as y,d as f,n,r as M}from"../chunks/jDjavuwI.js";const E='{"title":"Echo","local":"echo","sections":[{"title":"Inference","local":"inference","sections":[],"depth":2},{"title":"EchoModularPipeline","local":"diffusers.EchoModularPipeline","sections":[],"depth":2},{"title":"EchoBlocks","local":"diffusers.EchoBlocks","sections":[],"depth":2}],"depth":1}';var C=f('<meta name="hf:doc:metadata"/>'),V=f(`<p></p> <!> <p><a href="https://github.com/jd-opensource/JoyAI-Echo" rel="nofollow">Echo</a> is a long-video generation model. It supports an optional clean
first frame, ordered image/audio memory slots, and a stochastic few-step Distribution Matching Distillation (DMD)
sampler.
The pipeline generates synchronized video and audio.</p> <p>Echo is implemented as a <a href="/docs/diffusers/pr_14159/en/api/modular_diffusers/pipeline#diffusers.ModularPipeline">ModularPipeline</a> so its text encoding, memory conditioning, stochastic DMD denoising, and
decoding blocks can be run as a complete workflow or composed independently.</p> <!> <p>Load the official <a href="https://huggingface.co/jdopensource/JoyAI-Echo" rel="nofollow">jdopensource/JoyAI-Echo</a> checkpoint directly.
It uses the text encoder and tokenizer from <a href="https://huggingface.co/google/gemma-3-12b-it" rel="nofollow">google/gemma-3-12b-it</a>.
The Gemma repository is gated, so accept its license and authenticate with Hugging Face before loading the pipeline.</p> <p>The released model uses 241 frames in its long-video example. The video RoPE coordinates remain at the training rate
of 24 fps, independently of the output container rate.</p> <!> <p>The default DMD sigma schedule is the released eight-step schedule. It predicts <code>x0</code> at every step and re-noises with
fresh Gaussian noise at the next sigma, so a seeded <code>torch.Generator</code> controls both the initial noise and all
intermediate re-noising.</p> <p>Raw audio-memory encoding requires <code>torchaudio</code>. For reference parity, keep <code>audio_vae</code> in FP32 as shown above.
Modular workflows can cache the VAE encoder’s normalized, unpacked tensors. Video latents have shape <code>(batch, channels, frames, height, width)</code> and audio latents have shape <code>(batch, channels, time, mel_bins)</code>.
The core <code>denoise</code> block packs those tensors, expands conditioning for <code>num_videos_per_prompt</code>, and unpacks
denoised outputs back to the same VAE form. Pass initial <code>latents</code> and <code>audio_latents</code> in that unpacked form too.
Decoders take normalized VAE tensors and denormalize immediately before
decoding. <code>output=["latents", "audio_latents"]</code> returns normalized VAE tensors. With <code>output_type="latent"</code> and <code>output=["videos", "audio"]</code>, you get the denormalized VAE tensors without decoding.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A ModularPipeline for the Echo long-video DMD checkpoint.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Echo reference-to-video generation with clean first-frame conditioning, ordered image/audio memory slots, and
stochastic DMD denoising.</p> <p>Components:
text_encoder (<code>PreTrainedModel</code>) tokenizer (<code>PreTrainedTokenizerBase</code>) connectors (<code>LTX2TextConnectors</code>) vae
(<code>AutoencoderKLLTX2Video</code>) audio_vae (<code>AutoencoderKLLTX2Audio</code>) transformer (<code>LTX2VideoTransformer3DModel</code>)
video_processor (<code>VideoProcessor</code>) vocoder (<code>LTX2Vocoder</code>)</p> <p>Inputs:
prompt (<code>str</code>):
The prompt or prompts to guide image generation.
max_sequence_length (<code>int</code>, <em>optional</em>, defaults to 1024):
Maximum sequence length for prompt encoding.
image (<code>Image | Tensor</code>, <em>optional</em>):
Optional single first frame used as a clean reference condition.
memory_images (<code>list</code>, <em>optional</em>):
Ordered reference images, one per Echo memory slot.
memory_audio_waveforms (<code>list</code>, <em>optional</em>):
Ordered memory waveforms as <code>(channels, samples)</code> tensors. Inputs longer than 9.62 seconds are cropped to
their highest-response window. Use <code>None</code> for a silent slot.
memory_audio_sample_rates (<code>int | list</code>, <em>optional</em>):
Sampling rate shared by all memory waveforms, or one rate per slot.
height (<code>int</code>, <em>optional</em>, defaults to 512):
The height in pixels of the generated image.
width (<code>int</code>, <em>optional</em>, defaults to 704):
The width in pixels of the generated image.
model_frame_rate (<code>float</code>, <em>optional</em>, defaults to 24.0):
Training-time frame rate used for Echo video RoPE coordinates.
memory_position_offset (<code>float</code>, <em>optional</em>, defaults to 500.0):
Temporal center assigned to the first memory slot.
memory_position_slot_stride (<code>float</code>, <em>optional</em>, defaults to 50.0):
Temporal distance between consecutive memory-slot centers.
num_videos_per_prompt (<code>int</code>, <em>optional</em>, defaults to 1):
Number of videos per prompt.
num_frames (<code>int</code>, <em>optional</em>, defaults to 241):
Number of generated pixel frames; must be <code>1 + k * vae_temporal_compression_ratio</code>.
frame_rate (<code>float</code>, <em>optional</em>, defaults to 25.0):
Frame rate of the generated video.
latents (<code>Tensor</code>, <em>optional</em>):
Optional initial video noise in VAE form (B, C, F, H, W).
audio_latents (<code>Tensor</code>, <em>optional</em>):
Optional initial audio noise in VAE form (B, C, L, M).
generator (<code>Generator</code>, <em>optional</em>):
Torch generator for deterministic generation.
sigmas (<code>list | tuple</code>):
DMD sigma schedule, including the terminal zero.
attention_kwargs (<code>dict</code>, <em>optional</em>):
Additional kwargs for attention processors.
output_type (<code>str</code>, <em>optional</em>, defaults to pil):
Output format: ‘pil’, ‘np’, ‘pt’.
decode_timestep (<code>None</code>, <em>optional</em>, defaults to 0.0):
Timestep used to decode the final latents.
decode_noise_scale (<code>None</code>, <em>optional</em>):
Noise interpolation factor applied at the decode timestep.</p> <p>Outputs:
videos (<code>list</code>):
The generated videos.
audio (<code>Tensor</code>):
The generated audio waveform.</p></div> <!> <p></p>`,1);function N(g,b){k(b,!1),B(()=>{new URLSearchParams(window.location.search).get("fw")}),J();var t=V();j("rbrdnd",p=>{var m=C();W(m,"content",E),h(p,m)});var l=e(Z(t),2);o(l,{title:"Echo",local:"echo",headingTag:"h1"});var i=e(l,6);o(i,{title:"Inference",local:"inference",headingTag:"h2"});var d=e(i,6);T(d,{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaGF1ZGlvJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMENvbXBvbmVudHNNYW5hZ2VyJTJDJTIwTW9kdWxhclBpcGVsaW5lJTBBZnJvbSUyMGRpZmZ1c2Vycy51dGlscyUyMGltcG9ydCUyMGVuY29kZV92aWRlbyUwQSUwQW1hbmFnZXIlMjAlM0QlMjBDb21wb25lbnRzTWFuYWdlcigpJTBBcGlwZSUyMCUzRCUyME1vZHVsYXJQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIyamRvcGVuc291cmNlJTJGSm95QUktRWNobyUyMiUyQyUyMGNvbXBvbmVudHNfbWFuYWdlciUzRG1hbmFnZXIpJTBBcGlwZS5sb2FkX2NvbXBvbmVudHMoZHR5cGUlM0QlN0IlMjJkZWZhdWx0JTIyJTNBJTIwdG9yY2guYmZsb2F0MTYlMkMlMjAlMjJhdWRpb192YWUlMjIlM0ElMjB0b3JjaC5mbG9hdDMyJTdEKSUwQW1hbmFnZXIuZW5hYmxlX2F1dG9fY3B1X29mZmxvYWQoZGV2aWNlJTNEJTIyY3VkYSUyMiklMEFwaXBlLnZhZS5lbmFibGVfdGlsaW5nKCklMEElMEFmaXJzdF9mcmFtZSUyMCUzRCUyMEltYWdlLm9wZW4oJTIyZmlyc3RfZnJhbWUucG5nJTIyKS5jb252ZXJ0KCUyMlJHQiUyMiklMEFtZW1vcnlfaW1hZ2VzJTIwJTNEJTIwJTVCSW1hZ2Uub3BlbihwYXRoKS5jb252ZXJ0KCUyMlJHQiUyMiklMjBmb3IlMjBwYXRoJTIwaW4lMjAlNUIlMjJtZW1vcnlfMC5wbmclMjIlMkMlMjAlMjJtZW1vcnlfMS5wbmclMjIlNUQlNUQlMEFtZW1vcnlfYXVkaW9fd2l0aF9yYXRlcyUyMCUzRCUyMCU1QnRvcmNoYXVkaW8ubG9hZChwYXRoKSUyMGZvciUyMHBhdGglMjBpbiUyMCU1QiUyMm1lbW9yeV8wLndhdiUyMiUyQyUyMCUyMm1lbW9yeV8xLndhdiUyMiU1RCU1RCUwQW1lbW9yeV9hdWRpbyUyMCUzRCUyMCU1QndhdmVmb3JtJTIwZm9yJTIwd2F2ZWZvcm0lMkMlMjBfJTIwaW4lMjBtZW1vcnlfYXVkaW9fd2l0aF9yYXRlcyU1RCUwQW1lbW9yeV9hdWRpb19yYXRlcyUyMCUzRCUyMCU1QnNhbXBsZV9yYXRlJTIwZm9yJTIwXyUyQyUyMHNhbXBsZV9yYXRlJTIwaW4lMjBtZW1vcnlfYXVkaW9fd2l0aF9yYXRlcyU1RCUwQSUwQW91dHB1dCUyMCUzRCUyMHBpcGUoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEJTIyQSUyMGNpbmVtYXRpYyUyMGRpYWxvZ3VlJTIwc2NlbmUlMjBpbiUyMGElMjBxdWlldCUyMGNhZmUuJTIyJTJDJTBBJTIwJTIwJTIwJTIwaW1hZ2UlM0RmaXJzdF9mcmFtZSUyQyUwQSUyMCUyMCUyMCUyMG1lbW9yeV9pbWFnZXMlM0RtZW1vcnlfaW1hZ2VzJTJDJTBBJTIwJTIwJTIwJTIwbWVtb3J5X2F1ZGlvX3dhdmVmb3JtcyUzRG1lbW9yeV9hdWRpbyUyQyUwQSUyMCUyMCUyMCUyMG1lbW9yeV9hdWRpb19zYW1wbGVfcmF0ZXMlM0RtZW1vcnlfYXVkaW9fcmF0ZXMlMkMlMEElMjAlMjAlMjAlMjB3aWR0aCUzRDEyODAlMkMlMEElMjAlMjAlMjAlMjBoZWlnaHQlM0Q3MzYlMkMlMEElMjAlMjAlMjAlMjBudW1fZnJhbWVzJTNEMjQxJTJDJTBBJTIwJTIwJTIwJTIwZnJhbWVfcmF0ZSUzRDI1LjAlMkMlMEElMjAlMjAlMjAlMjBtb2RlbF9mcmFtZV9yYXRlJTNEMjQuMCUyQyUwQSUyMCUyMCUyMCUyMGdlbmVyYXRvciUzRHRvcmNoLkdlbmVyYXRvcihkZXZpY2UlM0QlMjJjdWRhJTIyKS5tYW51YWxfc2VlZCg0MiklMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfdHlwZSUzRCUyMm5wJTIyJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0JTNEJTVCJTIydmlkZW9zJTIyJTJDJTIwJTIyYXVkaW8lMjIlNUQlMkMlMEEpJTBBJTBBZW5jb2RlX3ZpZGVvKCUwQSUyMCUyMCUyMCUyMG91dHB1dCU1QiUyMnZpZGVvcyUyMiU1RCU1QjAlNUQlMkMlMEElMjAlMjAlMjAlMjBmcHMlM0QyNSUyQyUwQSUyMCUyMCUyMCUyMGF1ZGlvJTNEb3V0cHV0JTVCJTIyYXVkaW8lMjIlNUQlNUIwJTVELmZsb2F0KCkuY3B1KCklMkMlMEElMjAlMjAlMjAlMjBhdWRpb19zYW1wbGVfcmF0ZSUzRHBpcGUudm9jb2Rlci5jb25maWcub3V0cHV0X3NhbXBsaW5nX3JhdGUlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfcGF0aCUzRCUyMmVjaG8ubXA0JTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torchaudio
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ComponentsManager, ModularPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video
manager = ComponentsManager()
pipe = ModularPipeline.from_pretrained(<span class="hljs-string">&quot;jdopensource/JoyAI-Echo&quot;</span>, components_manager=manager)
pipe.load_components(dtype={<span class="hljs-string">&quot;default&quot;</span>: torch.bfloat16, <span class="hljs-string">&quot;audio_vae&quot;</span>: torch.float32})
manager.enable_auto_cpu_offload(device=<span class="hljs-string">&quot;cuda&quot;</span>)
pipe.vae.enable_tiling()
first_frame = Image.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;first_frame.png&quot;</span>).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
memory_images = [Image.<span class="hljs-built_in">open</span>(path).convert(<span class="hljs-string">&quot;RGB&quot;</span>) <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;memory_0.png&quot;</span>, <span class="hljs-string">&quot;memory_1.png&quot;</span>]]
memory_audio_with_rates = [torchaudio.load(path) <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;memory_0.wav&quot;</span>, <span class="hljs-string">&quot;memory_1.wav&quot;</span>]]
memory_audio = [waveform <span class="hljs-keyword">for</span> waveform, _ <span class="hljs-keyword">in</span> memory_audio_with_rates]
memory_audio_rates = [sample_rate <span class="hljs-keyword">for</span> _, sample_rate <span class="hljs-keyword">in</span> memory_audio_with_rates]
output = pipe(
    prompt=<span class="hljs-string">&quot;A cinematic dialogue scene in a quiet cafe.&quot;</span>,
    image=first_frame,
    memory_images=memory_images,
    memory_audio_waveforms=memory_audio,
    memory_audio_sample_rates=memory_audio_rates,
    width=<span class="hljs-number">1280</span>,
    height=<span class="hljs-number">736</span>,
    num_frames=<span class="hljs-number">241</span>,
    frame_rate=<span class="hljs-number">25.0</span>,
    model_frame_rate=<span class="hljs-number">24.0</span>,
    generator=torch.Generator(device=<span class="hljs-string">&quot;cuda&quot;</span>).manual_seed(<span class="hljs-number">42</span>),
    output_type=<span class="hljs-string">&quot;np&quot;</span>,
    output=[<span class="hljs-string">&quot;videos&quot;</span>, <span class="hljs-string">&quot;audio&quot;</span>],
)
encode_video(
    output[<span class="hljs-string">&quot;videos&quot;</span>][<span class="hljs-number">0</span>],
    fps=<span class="hljs-number">25</span>,
    audio=output[<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-number">0</span>].<span class="hljs-built_in">float</span>().cpu(),
    audio_sample_rate=pipe.vocoder.config.output_sampling_rate,
    output_path=<span class="hljs-string">&quot;echo.mp4&quot;</span>,
)`,lang:"py",wrap:!1});var r=e(d,6);o(r,{title:"EchoModularPipeline",local:"diffusers.EchoModularPipeline",headingTag:"h2"});var a=e(r,2),_=y(a);u(_,{name:"class diffusers.EchoModularPipeline",anchor:"diffusers.EchoModularPipeline",source:"https://github.com/huggingface/diffusers/blob/vr_14159/src/diffusers/modular_pipelines/echo/modular_pipeline.py#L27",parameters:[{name:"blocks",val:": diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None"},{name:"pretrained_model_name_or_path",val:": str | os.PathLike | None = None"},{name:"components_manager",val:": diffusers.modular_pipelines.components_manager.ComponentsManager | None = None"},{name:"collection",val:": str | None = None"},{name:"workflow",val:": str | None = None"},{name:"modular_config_dict",val:": dict[str, typing.Any] | None = None"},{name:"config_dict",val:": dict[str, typing.Any] | None = None"},{name:"**kwargs",val:""}]}),n(2),M(a);var c=e(a,2);o(c,{title:"EchoBlocks",local:"diffusers.EchoBlocks",headingTag:"h2"});var s=e(c,2),U=y(s);u(U,{name:"class diffusers.EchoBlocks",anchor:"diffusers.EchoBlocks",source:"https://github.com/huggingface/diffusers/blob/vr_14159/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py#L220",parameters:[]}),n(8),M(s);var w=e(s,2);v(w,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/pipelines/echo.md"}),n(2),h(g,t),I()}export{N as component};

Xet Storage Details

Size:: 14.4 kB
Xet hash:: 1aa9dd11ed6432c616748bb6858cf726c45a70066134ccc29cd82ccf11715574

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	import"../chunks/DsnmJJEf.js";import{i as J,h as j,H as o,a as T,D as u,E as v,s as W}from"../chunks/BLbO27j4.js";import{p as k,o as B,s as e,f as Z,a as h,b as I,c as y,d as f,n,r as M}from"../chunks/jDjavuwI.js";const E='{"title":"Echo","local":"echo","sections":[{"title":"Inference","local":"inference","sections":[],"depth":2},{"title":"EchoModularPipeline","local":"diffusers.EchoModularPipeline","sections":[],"depth":2},{"title":"EchoBlocks","local":"diffusers.EchoBlocks","sections":[],"depth":2}],"depth":1}';var C=f('<meta name="hf:doc:metadata"/>'),V=f(`<p></p> <!> <p><a href="https://github.com/jd-opensource/JoyAI-Echo" rel="nofollow">Echo</a> is a long-video generation model. It supports an optional clean
	first frame, ordered image/audio memory slots, and a stochastic few-step Distribution Matching Distillation (DMD)
	sampler.
	The pipeline generates synchronized video and audio.</p> <p>Echo is implemented as a <a href="/docs/diffusers/pr_14159/en/api/modular_diffusers/pipeline#diffusers.ModularPipeline">ModularPipeline</a> so its text encoding, memory conditioning, stochastic DMD denoising, and
	decoding blocks can be run as a complete workflow or composed independently.</p> <!> <p>Load the official <a href="https://huggingface.co/jdopensource/JoyAI-Echo" rel="nofollow">jdopensource/JoyAI-Echo</a> checkpoint directly.
	It uses the text encoder and tokenizer from <a href="https://huggingface.co/google/gemma-3-12b-it" rel="nofollow">google/gemma-3-12b-it</a>.
	The Gemma repository is gated, so accept its license and authenticate with Hugging Face before loading the pipeline.</p> <p>The released model uses 241 frames in its long-video example. The video RoPE coordinates remain at the training rate
	of 24 fps, independently of the output container rate.</p> <!> <p>The default DMD sigma schedule is the released eight-step schedule. It predicts <code>x0</code> at every step and re-noises with
	fresh Gaussian noise at the next sigma, so a seeded <code>torch.Generator</code> controls both the initial noise and all
	intermediate re-noising.</p> <p>Raw audio-memory encoding requires <code>torchaudio</code>. For reference parity, keep <code>audio_vae</code> in FP32 as shown above.
	Modular workflows can cache the VAE encoder’s normalized, unpacked tensors. Video latents have shape <code>(batch, channels, frames, height, width)</code> and audio latents have shape <code>(batch, channels, time, mel_bins)</code>.
	The core <code>denoise</code> block packs those tensors, expands conditioning for <code>num_videos_per_prompt</code>, and unpacks
	denoised outputs back to the same VAE form. Pass initial <code>latents</code> and <code>audio_latents</code> in that unpacked form too.
	Decoders take normalized VAE tensors and denormalize immediately before
	decoding. <code>output=["latents", "audio_latents"]</code> returns normalized VAE tensors. With <code>output_type="latent"</code> and <code>output=["videos", "audio"]</code>, you get the denormalized VAE tensors without decoding.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A ModularPipeline for the Echo long-video DMD checkpoint.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Echo reference-to-video generation with clean first-frame conditioning, ordered image/audio memory slots, and
	stochastic DMD denoising.</p> <p>Components:
	text_encoder (<code>PreTrainedModel</code>) tokenizer (<code>PreTrainedTokenizerBase</code>) connectors (<code>LTX2TextConnectors</code>) vae
	(<code>AutoencoderKLLTX2Video</code>) audio_vae (<code>AutoencoderKLLTX2Audio</code>) transformer (<code>LTX2VideoTransformer3DModel</code>)
	video_processor (<code>VideoProcessor</code>) vocoder (<code>LTX2Vocoder</code>)</p> <p>Inputs:
	prompt (<code>str</code>):
	The prompt or prompts to guide image generation.
	max_sequence_length (<code>int</code>, <em>optional</em>, defaults to 1024):
	Maximum sequence length for prompt encoding.
	image (<code>Image \| Tensor</code>, <em>optional</em>):
	Optional single first frame used as a clean reference condition.
	memory_images (<code>list</code>, <em>optional</em>):
	Ordered reference images, one per Echo memory slot.
	memory_audio_waveforms (<code>list</code>, <em>optional</em>):
	Ordered memory waveforms as <code>(channels, samples)</code> tensors. Inputs longer than 9.62 seconds are cropped to
	their highest-response window. Use <code>None</code> for a silent slot.
	memory_audio_sample_rates (<code>int \| list</code>, <em>optional</em>):
	Sampling rate shared by all memory waveforms, or one rate per slot.
	height (<code>int</code>, <em>optional</em>, defaults to 512):
	The height in pixels of the generated image.
	width (<code>int</code>, <em>optional</em>, defaults to 704):
	The width in pixels of the generated image.
	model_frame_rate (<code>float</code>, <em>optional</em>, defaults to 24.0):
	Training-time frame rate used for Echo video RoPE coordinates.
	memory_position_offset (<code>float</code>, <em>optional</em>, defaults to 500.0):
	Temporal center assigned to the first memory slot.
	memory_position_slot_stride (<code>float</code>, <em>optional</em>, defaults to 50.0):
	Temporal distance between consecutive memory-slot centers.
	num_videos_per_prompt (<code>int</code>, <em>optional</em>, defaults to 1):
	Number of videos per prompt.
	num_frames (<code>int</code>, <em>optional</em>, defaults to 241):
	Number of generated pixel frames; must be <code>1 + k * vae_temporal_compression_ratio</code>.
	frame_rate (<code>float</code>, <em>optional</em>, defaults to 25.0):
	Frame rate of the generated video.
	latents (<code>Tensor</code>, <em>optional</em>):
	Optional initial video noise in VAE form (B, C, F, H, W).
	audio_latents (<code>Tensor</code>, <em>optional</em>):
	Optional initial audio noise in VAE form (B, C, L, M).
	generator (<code>Generator</code>, <em>optional</em>):
	Torch generator for deterministic generation.
	sigmas (<code>list \| tuple</code>):
	DMD sigma schedule, including the terminal zero.
	attention_kwargs (<code>dict</code>, <em>optional</em>):
	Additional kwargs for attention processors.
	output_type (<code>str</code>, <em>optional</em>, defaults to pil):
	Output format: ‘pil’, ‘np’, ‘pt’.
	decode_timestep (<code>None</code>, <em>optional</em>, defaults to 0.0):
	Timestep used to decode the final latents.
	decode_noise_scale (<code>None</code>, <em>optional</em>):
	Noise interpolation factor applied at the decode timestep.</p> <p>Outputs:
	videos (<code>list</code>):
	The generated videos.
	audio (<code>Tensor</code>):
	The generated audio waveform.</p></div> <!> <p></p>`,1);function N(g,b){k(b,!1),B(()=>{new URLSearchParams(window.location.search).get("fw")}),J();var t=V();j("rbrdnd",p=>{var m=C();W(m,"content",E),h(p,m)});var l=e(Z(t),2);o(l,{title:"Echo",local:"echo",headingTag:"h1"});var i=e(l,6);o(i,{title:"Inference",local:"inference",headingTag:"h2"});var d=e(i,6);T(d,{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaGF1ZGlvJTBBZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMENvbXBvbmVudHNNYW5hZ2VyJTJDJTIwTW9kdWxhclBpcGVsaW5lJTBBZnJvbSUyMGRpZmZ1c2Vycy51dGlscyUyMGltcG9ydCUyMGVuY29kZV92aWRlbyUwQSUwQW1hbmFnZXIlMjAlM0QlMjBDb21wb25lbnRzTWFuYWdlcigpJTBBcGlwZSUyMCUzRCUyME1vZHVsYXJQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIyamRvcGVuc291cmNlJTJGSm95QUktRWNobyUyMiUyQyUyMGNvbXBvbmVudHNfbWFuYWdlciUzRG1hbmFnZXIpJTBBcGlwZS5sb2FkX2NvbXBvbmVudHMoZHR5cGUlM0QlN0IlMjJkZWZhdWx0JTIyJTNBJTIwdG9yY2guYmZsb2F0MTYlMkMlMjAlMjJhdWRpb192YWUlMjIlM0ElMjB0b3JjaC5mbG9hdDMyJTdEKSUwQW1hbmFnZXIuZW5hYmxlX2F1dG9fY3B1X29mZmxvYWQoZGV2aWNlJTNEJTIyY3VkYSUyMiklMEFwaXBlLnZhZS5lbmFibGVfdGlsaW5nKCklMEElMEFmaXJzdF9mcmFtZSUyMCUzRCUyMEltYWdlLm9wZW4oJTIyZmlyc3RfZnJhbWUucG5nJTIyKS5jb252ZXJ0KCUyMlJHQiUyMiklMEFtZW1vcnlfaW1hZ2VzJTIwJTNEJTIwJTVCSW1hZ2Uub3BlbihwYXRoKS5jb252ZXJ0KCUyMlJHQiUyMiklMjBmb3IlMjBwYXRoJTIwaW4lMjAlNUIlMjJtZW1vcnlfMC5wbmclMjIlMkMlMjAlMjJtZW1vcnlfMS5wbmclMjIlNUQlNUQlMEFtZW1vcnlfYXVkaW9fd2l0aF9yYXRlcyUyMCUzRCUyMCU1QnRvcmNoYXVkaW8ubG9hZChwYXRoKSUyMGZvciUyMHBhdGglMjBpbiUyMCU1QiUyMm1lbW9yeV8wLndhdiUyMiUyQyUyMCUyMm1lbW9yeV8xLndhdiUyMiU1RCU1RCUwQW1lbW9yeV9hdWRpbyUyMCUzRCUyMCU1QndhdmVmb3JtJTIwZm9yJTIwd2F2ZWZvcm0lMkMlMjBfJTIwaW4lMjBtZW1vcnlfYXVkaW9fd2l0aF9yYXRlcyU1RCUwQW1lbW9yeV9hdWRpb19yYXRlcyUyMCUzRCUyMCU1QnNhbXBsZV9yYXRlJTIwZm9yJTIwXyUyQyUyMHNhbXBsZV9yYXRlJTIwaW4lMjBtZW1vcnlfYXVkaW9fd2l0aF9yYXRlcyU1RCUwQSUwQW91dHB1dCUyMCUzRCUyMHBpcGUoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEJTIyQSUyMGNpbmVtYXRpYyUyMGRpYWxvZ3VlJTIwc2NlbmUlMjBpbiUyMGElMjBxdWlldCUyMGNhZmUuJTIyJTJDJTBBJTIwJTIwJTIwJTIwaW1hZ2UlM0RmaXJzdF9mcmFtZSUyQyUwQSUyMCUyMCUyMCUyMG1lbW9yeV9pbWFnZXMlM0RtZW1vcnlfaW1hZ2VzJTJDJTBBJTIwJTIwJTIwJTIwbWVtb3J5X2F1ZGlvX3dhdmVmb3JtcyUzRG1lbW9yeV9hdWRpbyUyQyUwQSUyMCUyMCUyMCUyMG1lbW9yeV9hdWRpb19zYW1wbGVfcmF0ZXMlM0RtZW1vcnlfYXVkaW9fcmF0ZXMlMkMlMEElMjAlMjAlMjAlMjB3aWR0aCUzRDEyODAlMkMlMEElMjAlMjAlMjAlMjBoZWlnaHQlM0Q3MzYlMkMlMEElMjAlMjAlMjAlMjBudW1fZnJhbWVzJTNEMjQxJTJDJTBBJTIwJTIwJTIwJTIwZnJhbWVfcmF0ZSUzRDI1LjAlMkMlMEElMjAlMjAlMjAlMjBtb2RlbF9mcmFtZV9yYXRlJTNEMjQuMCUyQyUwQSUyMCUyMCUyMCUyMGdlbmVyYXRvciUzRHRvcmNoLkdlbmVyYXRvcihkZXZpY2UlM0QlMjJjdWRhJTIyKS5tYW51YWxfc2VlZCg0MiklMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfdHlwZSUzRCUyMm5wJTIyJTJDJTBBJTIwJTIwJTIwJTIwb3V0cHV0JTNEJTVCJTIydmlkZW9zJTIyJTJDJTIwJTIyYXVkaW8lMjIlNUQlMkMlMEEpJTBBJTBBZW5jb2RlX3ZpZGVvKCUwQSUyMCUyMCUyMCUyMG91dHB1dCU1QiUyMnZpZGVvcyUyMiU1RCU1QjAlNUQlMkMlMEElMjAlMjAlMjAlMjBmcHMlM0QyNSUyQyUwQSUyMCUyMCUyMCUyMGF1ZGlvJTNEb3V0cHV0JTVCJTIyYXVkaW8lMjIlNUQlNUIwJTVELmZsb2F0KCkuY3B1KCklMkMlMEElMjAlMjAlMjAlMjBhdWRpb19zYW1wbGVfcmF0ZSUzRHBpcGUudm9jb2Rlci5jb25maWcub3V0cHV0X3NhbXBsaW5nX3JhdGUlMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRfcGF0aCUzRCUyMmVjaG8ubXA0JTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
	<span class="hljs-keyword">import</span> torchaudio
	<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

	<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ComponentsManager, ModularPipeline
	<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video

	manager = ComponentsManager()
	pipe = ModularPipeline.from_pretrained(<span class="hljs-string">"jdopensource/JoyAI-Echo"</span>, components_manager=manager)
	pipe.load_components(dtype={<span class="hljs-string">"default"</span>: torch.bfloat16, <span class="hljs-string">"audio_vae"</span>: torch.float32})
	manager.enable_auto_cpu_offload(device=<span class="hljs-string">"cuda"</span>)
	pipe.vae.enable_tiling()

	first_frame = Image.<span class="hljs-built_in">open</span>(<span class="hljs-string">"first_frame.png"</span>).convert(<span class="hljs-string">"RGB"</span>)
	memory_images = [Image.<span class="hljs-built_in">open</span>(path).convert(<span class="hljs-string">"RGB"</span>) <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> [<span class="hljs-string">"memory_0.png"</span>, <span class="hljs-string">"memory_1.png"</span>]]
	memory_audio_with_rates = [torchaudio.load(path) <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> [<span class="hljs-string">"memory_0.wav"</span>, <span class="hljs-string">"memory_1.wav"</span>]]
	memory_audio = [waveform <span class="hljs-keyword">for</span> waveform, _ <span class="hljs-keyword">in</span> memory_audio_with_rates]
	memory_audio_rates = [sample_rate <span class="hljs-keyword">for</span> _, sample_rate <span class="hljs-keyword">in</span> memory_audio_with_rates]

	output = pipe(
	prompt=<span class="hljs-string">"A cinematic dialogue scene in a quiet cafe."</span>,
	image=first_frame,
	memory_images=memory_images,
	memory_audio_waveforms=memory_audio,
	memory_audio_sample_rates=memory_audio_rates,
	width=<span class="hljs-number">1280</span>,
	height=<span class="hljs-number">736</span>,
	num_frames=<span class="hljs-number">241</span>,
	frame_rate=<span class="hljs-number">25.0</span>,
	model_frame_rate=<span class="hljs-number">24.0</span>,
	generator=torch.Generator(device=<span class="hljs-string">"cuda"</span>).manual_seed(<span class="hljs-number">42</span>),
	output_type=<span class="hljs-string">"np"</span>,
	output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"audio"</span>],
	)

	encode_video(
	output[<span class="hljs-string">"videos"</span>][<span class="hljs-number">0</span>],
	fps=<span class="hljs-number">25</span>,
	audio=output[<span class="hljs-string">"audio"</span>][<span class="hljs-number">0</span>].<span class="hljs-built_in">float</span>().cpu(),
	audio_sample_rate=pipe.vocoder.config.output_sampling_rate,
	output_path=<span class="hljs-string">"echo.mp4"</span>,
	)`,lang:"py",wrap:!1});var r=e(d,6);o(r,{title:"EchoModularPipeline",local:"diffusers.EchoModularPipeline",headingTag:"h2"});var a=e(r,2),_=y(a);u(_,{name:"class diffusers.EchoModularPipeline",anchor:"diffusers.EchoModularPipeline",source:"https://github.com/huggingface/diffusers/blob/vr_14159/src/diffusers/modular_pipelines/echo/modular_pipeline.py#L27",parameters:[{name:"blocks",val:": diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks \| None = None"},{name:"pretrained_model_name_or_path",val:": str \| os.PathLike \| None = None"},{name:"components_manager",val:": diffusers.modular_pipelines.components_manager.ComponentsManager \| None = None"},{name:"collection",val:": str \| None = None"},{name:"workflow",val:": str \| None = None"},{name:"modular_config_dict",val:": dict[str, typing.Any] \| None = None"},{name:"config_dict",val:": dict[str, typing.Any] \| None = None"},{name:"**kwargs",val:""}]}),n(2),M(a);var c=e(a,2);o(c,{title:"EchoBlocks",local:"diffusers.EchoBlocks",headingTag:"h2"});var s=e(c,2),U=y(s);u(U,{name:"class diffusers.EchoBlocks",anchor:"diffusers.EchoBlocks",source:"https://github.com/huggingface/diffusers/blob/vr_14159/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py#L220",parameters:[]}),n(8),M(s);var w=e(s,2);v(w,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/pipelines/echo.md"}),n(2),h(g,t),I()}export{N as component};