diffusers/pr_13881/en/_app/immutable/nodes/118.D9zUb2wS.js

import"../chunks/DsnmJJEf.js";import{i as N,h as w,C as q,H as s,D as l,E as D,s as M}from"../chunks/CmJXCtRL.js";import{p as k,o as L,s as e,f as T,a as p,b as S,c as r,d as _,n as o,r as i}from"../chunks/DK803DsY.js";const A='{"title":"Parallelism","local":"parallelism","sections":[{"title":"ParallelConfig","local":"diffusers.ParallelConfig","sections":[],"depth":2},{"title":"ContextParallelConfig","local":"diffusers.ContextParallelConfig","sections":[],"depth":2}],"depth":1}';var F=_('<meta name="hf:doc:metadata"/>'),E=_('<p></p> <!> <!> <p>Parallelism strategies help speed up diffusion transformers by distributing computations across multiple devices, allowing for faster inference/training times. Refer to the <a href="../training/distributed_inference">Distributed inferece</a> guide to learn more.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for applying different parallelisms.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for context parallelism.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Apply context parallel on a model.</p></div> <!> <p></p>',1);function z(v,b){k(b,!1),L(()=>{new URLSearchParams(window.location.search).get("fw")}),N();var d=E();w("1o3ncvv",u=>{var h=F();M(h,"content",A),p(u,h)});var c=e(T(d),2);q(c,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var m=e(c,2);s(m,{title:"Parallelism",local:"parallelism",headingTag:"h1"});var g=e(m,4);s(g,{title:"ParallelConfig",local:"diffusers.ParallelConfig",headingTag:"h2"});var n=e(g,2),y=r(n);l(y,{name:"class diffusers.ParallelConfig",anchor:"diffusers.ParallelConfig",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/models/_modeling_parallel.py#L158",parameters:[{name:"context_parallel_config",val:": diffusers.models._modeling_parallel.ContextParallelConfig | None = None"},{name:"_rank",val:": int = None"},{name:"_world_size",val:": int = None"},{name:"_device",val:": device = None"},{name:"_mesh",val:": DeviceMesh = None"}],parametersDescription:[{anchor:"diffusers.ParallelConfig.context_parallel_config",description:`<strong>context_parallel_config</strong> (<code>ContextParallelConfig</code>, <em>optional</em>) &#x2014;
Configuration for context parallelism.`,name:"context_parallel_config"}]}),o(2),i(n);var f=e(n,2);s(f,{title:"ContextParallelConfig",local:"diffusers.ContextParallelConfig",headingTag:"h2"});var t=e(f,2),x=r(t);l(x,{name:"class diffusers.ContextParallelConfig",anchor:"diffusers.ContextParallelConfig",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/models/_modeling_parallel.py#L42",parameters:[{name:"ring_degree",val:": int | None = None"},{name:"ulysses_degree",val:": int | None = None"},{name:"convert_to_fp32",val:": bool = True"},{name:"rotate_method",val:": typing.Literal['allgather', 'alltoall'] = 'allgather'"},{name:"mesh",val:": torch.distributed.device_mesh.DeviceMesh | None = None"},{name:"ulysses_anything",val:": bool = False"},{name:"ring_anything",val:": bool = False"},{name:"_rank",val:": int = None"},{name:"_world_size",val:": int = None"},{name:"_device",val:": device = None"},{name:"_mesh",val:": DeviceMesh = None"},{name:"_flattened_mesh",val:": DeviceMesh = None"},{name:"_ring_mesh",val:": DeviceMesh = None"},{name:"_ulysses_mesh",val:": DeviceMesh = None"},{name:"_ring_local_rank",val:": int = None"},{name:"_ulysses_local_rank",val:": int = None"}],parametersDescription:[{anchor:"diffusers.ContextParallelConfig.ring_degree",description:`<strong>ring_degree</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of devices to use for Ring Attention. Sequence is split across devices. Each device computes
attention between its local Q and KV chunks passed sequentially around ring. Lower memory (only holds 1/N
of KV at a time), overlaps compute with communication, but requires N iterations to see all tokens. Best
for long sequences with limited memory/bandwidth. Number of devices to use for ring attention within a
context parallel region. Must be a divisor of the total number of devices in the context parallel mesh.`,name:"ring_degree"},{anchor:"diffusers.ContextParallelConfig.ulysses_degree",description:`<strong>ulysses_degree</strong> (<code>int</code>, <em>optional</em>, defaults to <code>1</code>) &#x2014;
Number of devices to use for Ulysses Attention. Sequence split is across devices. Each device computes
local QKV, then all-gathers all KV chunks to compute full attention in one pass. Higher memory (stores all
KV), requires high-bandwidth all-to-all communication, but lower latency. Best for moderate sequences with
good interconnect bandwidth.`,name:"ulysses_degree"},{anchor:"diffusers.ContextParallelConfig.convert_to_fp32",description:`<strong>convert_to_fp32</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to convert output and LSE to float32 for ring attention numerical stability.`,name:"convert_to_fp32"},{anchor:"diffusers.ContextParallelConfig.rotate_method",description:`<strong>rotate_method</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;allgather&quot;</code>) &#x2014;
Method to use for rotating key/value states across devices in ring attention. Currently, only <code>&quot;allgather&quot;</code>
is supported.`,name:"rotate_method"},{anchor:"diffusers.ContextParallelConfig.ulysses_anything",description:`<strong>ulysses_anything</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to enable &#x201C;Ulysses Anything&#x201D; mode, which supports arbitrary sequence lengths and head counts that
are not evenly divisible by <code>ulysses_degree</code>. When enabled, <code>ulysses_degree</code> must be greater than 1 and
<code>ring_degree</code> must be 1.`,name:"ulysses_anything"},{anchor:"diffusers.ContextParallelConfig.ring_anything",description:`<strong>ring_anything</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to enable &#x201C;Ring Anything&#x201D; mode, which supports arbitrary sequence lengths. When enabled,
<code>ring_degree</code> must be greater than 1 and <code>ulysses_degree</code> must be 1.`,name:"ring_anything"},{anchor:"diffusers.ContextParallelConfig.mesh",description:`<strong>mesh</strong> (<code>torch.distributed.device_mesh.DeviceMesh</code>, <em>optional</em>) &#x2014;
A custom device mesh to use for context parallelism. If provided, this mesh will be used instead of
creating a new one. This is useful when combining context parallelism with other parallelism strategies
(e.g., FSDP, tensor parallelism) that share the same device mesh. The mesh must have both &#x201C;ring&#x201D; and
&#x201C;ulysses&#x201D; dimensions. Use size 1 for dimensions not being used (e.g., <code>mesh_shape=(2, 1, 4)</code> with
<code>mesh_dim_names=(&quot;ring&quot;, &quot;ulysses&quot;, &quot;fsdp&quot;)</code> for ring attention only with FSDP).`,name:"mesh"}]}),o(2),i(t);var a=e(t,2),C=r(a);l(C,{name:"diffusers.hooks.apply_context_parallel",anchor:"diffusers.hooks.apply_context_parallel",source:"https://github.com/huggingface/diffusers/blob/vr_13881/src/diffusers/hooks/context_parallel.py#L80",parameters:[{name:"module",val:": Module"},{name:"parallel_config",val:": ContextParallelConfig"},{name:"plan",val:": dict"}]}),o(2),i(a);var P=e(a,2);D(P,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/parallel.md"}),o(2),p(v,d),S()}export{z as component};