Upload folder using huggingface_hub

.gitattributes

@@ -903,3 +903,119 @@ models/text_encoders/models--openai--clip-vit-large-patch14/blobs/a2bf730a0c7deb
 models/text_encoders/models--xlabs-ai--xflux_text_encoders/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00 filter=lfs diff=lfs merge=lfs -text
 models/text_encoders/models--xlabs-ai--xflux_text_encoders/blobs/d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86 filter=lfs diff=lfs merge=lfs -text
 models/text_encoders/models--xlabs-ai--xflux_text_encoders/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-Hunyuan3DWrapper/hy3dgen/texgen/custom_rasterizer/dist/custom_rasterizer-0.1.0+torch241.cuda121-py3.10-linux-x86_64.egg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/end.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/fox.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/jeep.mp4 filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/ltxvideo-first-sequence-conditioning.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/ltxvideo-frame-interpolation.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/ltxvideo-i2v-distilled.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/ltxvideo-last-sequence-conditioning.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/shrek2.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/assets/start.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/ltxvideo-flow-edit.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/ltxvideo-rf-edit.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/ref.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/shot.mp4 filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/shot2.mp4 filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/shrek2.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-LTXVideo/tricks/assets/shrek3.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-TogetherVision/images/Latest.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-TogetherVision/images/node-screenshot-old.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-TogetherVision/images/node-screenshot.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/00.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/01.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/02.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/03.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/04.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/05.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/06.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/07.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/08.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/09.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/10.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/11.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/12.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/13.gif filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/72105_388.mp4_00-00.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/72105_388.mp4_00-01.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/72109_125.mp4_00-00.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/72109_125.mp4_00-01.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/72110_255.mp4_00-00.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/72110_255.mp4_00-01.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/74302_1349_frame1.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/74302_1349_frame3.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/Japan_v2_1_070321_s3_frame1.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/Japan_v2_1_070321_s3_frame3.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/Japan_v2_2_062266_s2_frame1.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/Japan_v2_2_062266_s2_frame3.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/frame0001_05.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/frame0001_09.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/frame0001_10.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/assets/frame0016_10.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/ldm/modules/image_degradation/utils/test.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/prompts/512_interp/74906_1462_frame1.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/prompts/512_interp/74906_1462_frame3.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/prompts/512_interp/Japan_v2_2_062266_s2_frame1.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/prompts/512_interp/Japan_v2_2_062266_s2_frame3.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/prompts/512_interp/Japan_v2_3_119235_s2_frame1.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-ToonCrafter/ToonCrafter/prompts/512_interp/Japan_v2_3_119235_s2_frame3.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-UNO/asset/show.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-WanVideoWrapper/example_workflows/example_inputs/env.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-WanVideoWrapper/example_workflows/example_inputs/human.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-WanVideoWrapper/example_workflows/example_inputs/wolf_interpolated.mp4 filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-easycontrol/asset/show.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-easycontrol/asset/show_2.webp filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-fastblend/drop.wav filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-fastblend/jiji.wav filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-faster-whisper/workflows/faster_whisper_suttitle.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_DiffRhythm/g2p/sources/chinese_lexicon.txt filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_DiffRhythm/images/2025-03-16_03-53-48.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_InfiniteYou/assets/face_combine_workflow.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_InfiniteYou/assets/face_swap.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_InfiniteYou/assets/musk.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_InfiniteYou/assets/teaser.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_InfiniteYou/assets/workflow_example.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_Patches_ll/example/PuLID_with_FBcache.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_Patches_ll/example/PuLID_with_teacache.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_Patches_ll/example/workflow_base.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_Patches_ll/example/workflow_hunyuanvideo.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_Patches_ll/example/workflow_ltxvideo.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_TiledKSampler/examples/ComfyUI_02006_.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI_TiledKSampler/examples/ComfyUI_02010_.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/comfyui_LLM_party/custom_tool/img_temp/-1738351262.JPG filter=lfs diff=lfs merge=lfs -text
+custom_nodes/comfyui_controlnet_aux/ckpts/LayerNorm/DensePose-TorchScript-with-hint-image/densepose_r50_fpn_dl.torchscript filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/audio_split_stems.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/background_removal.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/face_styler.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/flux_dev.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/flux_inpainting.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/layer_diffusion.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/mars-id.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/ominicontrol.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/example_workflows/txt2img_SDXL.jpg filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/img_utils/depth.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/eden_comfy_pipelines/img_utils/image.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Agfa[[:space:]]Vista[[:space:]]200.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Anime.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/CandleLight.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/ColorNegative.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Fuji[[:space:]]Velvia[[:space:]]50.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Ilford[[:space:]]HP5.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Kodak[[:space:]]ColorPlus[[:space:]]200.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Kodak[[:space:]]Ektachrome[[:space:]]100.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Kodak[[:space:]]Gold[[:space:]]200.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Kodak[[:space:]]Kodachrome[[:space:]]64.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/Kodak[[:space:]]TRI-X[[:space:]]400.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/TealMagentaGold.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/broadcast.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/bw.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/clipped.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/dd.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/h8.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/lit.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/modern.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/preset.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/HaldCLUT/retro.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/noise.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/mikey_nodes/noise_bw.png filter=lfs diff=lfs merge=lfs -text

custom_nodes/ComfyUI-AdvancedLivePortrait/nodes.py

@@ -523,8 +523,8 @@ def logging_time(original_fn):
     return wrapper_fn
 
 
-#exp_data_dir = os.path.join(current_directory, "exp_data")
-exp_data_dir = os.path.join(folder_paths.output_directory, "exp_data")
+exp_data_dir = os.path.join(current_directory, "exp_data")
+# exp_data_dir = os.path.join(folder_paths.output_directory, "exp_data")
 if os.path.isdir(exp_data_dir) == False:
     os.mkdir(exp_data_dir)
 class SaveExpData:

custom_nodes/ComfyUI-CogVideoXWrapper/custom_cogvideox_transformer_3d.py

@@ -46,40 +46,9 @@ except:
 
 from comfy.ldm.modules.attention import optimized_attention
 
-
-def set_attention_func(attention_mode, heads):
-    if attention_mode == "sdpa" or attention_mode == "fused_sdpa":
-        def func(q, k, v, is_causal=False, attn_mask=None):
-            return F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask, dropout_p=0.0, is_causal=is_causal)
-        return func
-    elif attention_mode == "comfy":
-        def func(q, k, v, is_causal=False, attn_mask=None):
-            return optimized_attention(q, k, v, mask=attn_mask, heads=heads, skip_reshape=True)
-        return func
-
-    elif attention_mode == "sageattn" or attention_mode == "fused_sageattn":
-        @torch.compiler.disable()
-        def func(q, k, v, is_causal=False, attn_mask=None):
-            return sageattn(q, k, v, is_causal=is_causal, attn_mask=attn_mask)
-        return func
-    elif attention_mode == "sageattn_qk_int8_pv_fp16_cuda":
-        from sageattention import sageattn_qk_int8_pv_fp16_cuda
-        @torch.compiler.disable()
-        def func(q, k, v, is_causal=False, attn_mask=None):
-            return sageattn_qk_int8_pv_fp16_cuda(q, k, v, is_causal=is_causal, attn_mask=attn_mask, pv_accum_dtype="fp32")
-        return func
-    elif attention_mode == "sageattn_qk_int8_pv_fp16_triton":
-        from sageattention import sageattn_qk_int8_pv_fp16_triton
-        @torch.compiler.disable()
-        def func(q, k, v, is_causal=False, attn_mask=None):
-            return sageattn_qk_int8_pv_fp16_triton(q, k, v, is_causal=is_causal, attn_mask=attn_mask)
-        return func
-    elif attention_mode == "sageattn_qk_int8_pv_fp8_cuda":
-        from sageattention import sageattn_qk_int8_pv_fp8_cuda
-        @torch.compiler.disable()
-        def func(q, k, v, is_causal=False, attn_mask=None):
-            return sageattn_qk_int8_pv_fp8_cuda(q, k, v, is_causal=is_causal, attn_mask=attn_mask, pv_accum_dtype="fp32+fp32")
-        return func
+@torch.compiler.disable()
+def sageattn_func(query, key, value, attn_mask=None, dropout_p=0.0,is_causal=False):
+    return sageattn(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p,is_causal=is_causal)
 
 def fft(tensor):
     tensor_fft = torch.fft.fft2(tensor)
@@ -98,18 +67,16 @@ def fft(tensor):
 
     return low_freq_fft, high_freq_fft
 
-#region Attention
 class CogVideoXAttnProcessor2_0:
     r"""
     Processor for implementing scaled dot-product attention for the CogVideoX model. It applies a rotary embedding on
     query and key vectors, but does not include spatial normalization.
     """
 
-    def __init__(self, attn_func, attention_mode: Optional[str] = None):
+    def __init__(self):
         if not hasattr(F, "scaled_dot_product_attention"):
             raise ImportError("CogVideoXAttnProcessor requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
-        self.attention_mode = attention_mode
-        self.attn_func = attn_func
+        
     def __call__(
         self,
         attn: Attention,
@@ -117,6 +84,7 @@ class CogVideoXAttnProcessor2_0:
         encoder_hidden_states: torch.Tensor,
         attention_mask: Optional[torch.Tensor] = None,
         image_rotary_emb: Optional[torch.Tensor] = None,
+        attention_mode: Optional[str] = None,
     ) -> torch.Tensor:
         text_seq_length = encoder_hidden_states.size(1)
 
@@ -133,7 +101,7 @@ class CogVideoXAttnProcessor2_0:
         if attn.to_q.weight.dtype == torch.float16 or attn.to_q.weight.dtype == torch.bfloat16:
             hidden_states = hidden_states.to(attn.to_q.weight.dtype)
 
-        if not "fused" in self.attention_mode:
+        if attention_mode != "fused_sdpa" or attention_mode != "fused_sageattn":
             query = attn.to_q(hidden_states)
             key = attn.to_k(hidden_states)
             value = attn.to_v(hidden_states)
@@ -160,10 +128,16 @@ class CogVideoXAttnProcessor2_0:
             if not attn.is_cross_attention:
                 key[:, :, text_seq_length:] = apply_rotary_emb(key[:, :, text_seq_length:], image_rotary_emb)
                 
-        hidden_states = self.attn_func(query, key, value, attn_mask=attention_mask, is_causal=False)
-       
-        if self.attention_mode != "comfy":
+        if attention_mode == "sageattn" or attention_mode == "fused_sageattn":
+            hidden_states = sageattn_func(query, key, value, attn_mask=attention_mask, dropout_p=0.0,is_causal=False)
             hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        elif attention_mode == "sdpa" or attention_mode == "fused_sdpa":
+            hidden_states = F.scaled_dot_product_attention(
+                query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+                )
+            hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        elif attention_mode == "comfy":
+            hidden_states = optimized_attention(query, key, value, mask=attention_mask, heads=attn.heads, skip_reshape=True)
 
         # linear proj
         hidden_states = attn.to_out[0](hidden_states)
@@ -229,15 +203,13 @@ class CogVideoXBlock(nn.Module):
         ff_inner_dim: Optional[int] = None,
         ff_bias: bool = True,
         attention_out_bias: bool = True,
-        attention_mode: Optional[str] = "sdpa",
     ):
         super().__init__()
 
         # 1. Self Attention
         self.norm1 = CogVideoXLayerNormZero(time_embed_dim, dim, norm_elementwise_affine, norm_eps, bias=True)
-
-        attn_func = set_attention_func(attention_mode, num_attention_heads)
        
+
         self.attn1 = Attention(
             query_dim=dim,
             dim_head=attention_head_dim,
@@ -246,7 +218,7 @@ class CogVideoXBlock(nn.Module):
             eps=1e-6,
             bias=attention_bias,
             out_bias=attention_out_bias,
-            processor=CogVideoXAttnProcessor2_0(attn_func, attention_mode=attention_mode),
+            processor=CogVideoXAttnProcessor2_0(),
         )
 
         # 2. Feed Forward
@@ -275,6 +247,7 @@ class CogVideoXBlock(nn.Module):
         fastercache_counter=0,
         fastercache_start_step=15,
         fastercache_device="cuda:0",
+        attention_mode="sdpa",
     ) -> torch.Tensor:
         #print("hidden_states in block: ", hidden_states.shape) #1.5: torch.Size([2, 3200, 3072]) 10.: torch.Size([2, 6400, 3072])
         text_seq_length = encoder_hidden_states.size(1)
@@ -313,6 +286,7 @@ class CogVideoXBlock(nn.Module):
                     hidden_states=norm_hidden_states,
                     encoder_hidden_states=norm_encoder_hidden_states,
                     image_rotary_emb=image_rotary_emb,
+                    attention_mode=attention_mode,
                 )
                 if fastercache_counter == fastercache_start_step:
                     self.cached_hidden_states = [attn_hidden_states.to(fastercache_device), attn_hidden_states.to(fastercache_device)]
@@ -324,7 +298,8 @@ class CogVideoXBlock(nn.Module):
             attn_hidden_states, attn_encoder_hidden_states = self.attn1(
                 hidden_states=norm_hidden_states,
                 encoder_hidden_states=norm_encoder_hidden_states,
-                image_rotary_emb=image_rotary_emb
+                image_rotary_emb=image_rotary_emb,
+                attention_mode=attention_mode,
             )
         
         hidden_states = hidden_states + gate_msa * attn_hidden_states
@@ -433,7 +408,6 @@ class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
         use_rotary_positional_embeddings: bool = False,
         use_learned_positional_embeddings: bool = False,
         patch_bias: bool = True,
-        attention_mode: Optional[str] = "sdpa",
     ):
         super().__init__()
         inner_dim = num_attention_heads * attention_head_dim
@@ -487,7 +461,6 @@ class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
                     dropout=dropout,
                     activation_fn=activation_fn,
                     attention_bias=attention_bias,
-                    attention_mode=attention_mode,
                     norm_elementwise_affine=norm_elementwise_affine,
                     norm_eps=norm_eps,
                 )
@@ -523,12 +496,73 @@ class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
         self.fastercache_hf_step = 30
         self.fastercache_device = "cuda"
         self.fastercache_num_blocks_to_cache = len(self.transformer_blocks)
-        self.attention_mode = attention_mode
+        self.attention_mode = "sdpa"
         
 
     def _set_gradient_checkpointing(self, module, value=False):
         self.gradient_checkpointing = value
-    #region forward
+
+    @property
+    # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors
+    def attn_processors(self) -> Dict[str, AttentionProcessor]:
+        r"""
+        Returns:
+            `dict` of attention processors: A dictionary containing all attention processors used in the model with
+            indexed by its weight name.
+        """
+        # set recursively
+        processors = {}
+
+        def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]):
+            if hasattr(module, "get_processor"):
+                processors[f"{name}.processor"] = module.get_processor()
+
+            for sub_name, child in module.named_children():
+                fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+            return processors
+
+        for name, module in self.named_children():
+            fn_recursive_add_processors(name, module, processors)
+
+        return processors
+
+    # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor
+    def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]):
+        r"""
+        Sets the attention processor to use to compute attention.
+
+        Parameters:
+            processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+                The instantiated processor class or a dictionary of processor classes that will be set as the processor
+                for **all** `Attention` layers.
+
+                If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+                processor. This is strongly recommended when setting trainable attention processors.
+
+        """
+        count = len(self.attn_processors.keys())
+
+        if isinstance(processor, dict) and len(processor) != count:
+            raise ValueError(
+                f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+                f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+            )
+
+        def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+            if hasattr(module, "set_processor"):
+                if not isinstance(processor, dict):
+                    module.set_processor(processor)
+                else:
+                    module.set_processor(processor.pop(f"{name}.processor"))
+
+            for sub_name, child in module.named_children():
+                fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+        for name, module in self.named_children():
+            fn_recursive_attn_processor(name, module, processor)
+
+
     def forward(
         self,
         hidden_states: torch.Tensor,
@@ -590,7 +624,8 @@ class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
                     block_use_fastercache = i <= self.fastercache_num_blocks_to_cache,
                     fastercache_counter = self.fastercache_counter,
                     fastercache_start_step = self.fastercache_start_step,
-                    fastercache_device = self.fastercache_device
+                    fastercache_device = self.fastercache_device,
+                    attention_mode = self.attention_mode
                 )
 
                 if (controlnet_states is not None) and (i < len(controlnet_states)):
@@ -660,7 +695,8 @@ class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
                     block_use_fastercache = i <= self.fastercache_num_blocks_to_cache,
                     fastercache_counter = self.fastercache_counter,
                     fastercache_start_step = self.fastercache_start_step,
-                    fastercache_device = self.fastercache_device
+                    fastercache_device = self.fastercache_device,
+                    attention_mode = self.attention_mode
                 )
                 #has_nan = torch.isnan(hidden_states).any()
                 #if has_nan:
@@ -718,4 +754,4 @@ class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin, PeftAdapterMixin):
         if not return_dict:
             return (output,)
         return Transformer2DModelOutput(sample=output)
-    
+    

custom_nodes/ComfyUI-CogVideoXWrapper/examples/cogvideox_Fun_180_orbit_01.json

@@ -0,0 +1,1922 @@
+{
+  "last_node_id": 73,
+  "last_link_id": 165,
+  "nodes": [
+    {
+      "id": 20,
+      "type": "CLIPLoader",
+      "pos": {
+        "0": -27,
+        "1": 42
+      },
+      "size": {
+        "0": 451.30548095703125,
+        "1": 82
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "CLIP",
+          "type": "CLIP",
+          "links": [
+            54
+          ],
+          "slot_index": 0,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CLIPLoader"
+      },
+      "widgets_values": [
+        "t5\\google_t5-v1_1-xxl_encoderonly-fp8_e4m3fn.safetensors",
+        "sd3"
+      ]
+    },
+    {
+      "id": 52,
+      "type": "CogVideoLoraSelect",
+      "pos": {
+        "0": -3,
+        "1": -383
+      },
+      "size": [
+        438.44762263180314,
+        106
+      ],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "prev_lora",
+          "type": "COGLORA",
+          "link": null,
+          "shape": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "lora",
+          "type": "COGLORA",
+          "links": [
+            124
+          ]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoLoraSelect"
+      },
+      "widgets_values": [
+        "DimensionX_orbit_left_lora_rank256_bf16.safetensors",
+        1,
+        true
+      ]
+    },
+    {
+      "id": 55,
+      "type": "ImageFlip+",
+      "pos": {
+        "0": 1247,
+        "1": 770
+      },
+      "size": {
+        "0": 315,
+        "1": 58
+      },
+      "flags": {},
+      "order": 22,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 130
+        }
+      ],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            131,
+            151
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ImageFlip+"
+      },
+      "widgets_values": [
+        "x"
+      ]
+    },
+    {
+      "id": 54,
+      "type": "ImageFlip+",
+      "pos": {
+        "0": 847,
+        "1": 802
+      },
+      "size": {
+        "0": 315,
+        "1": 58
+      },
+      "flags": {},
+      "order": 10,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 128
+        }
+      ],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            129
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ImageFlip+"
+      },
+      "widgets_values": [
+        "x"
+      ]
+    },
+    {
+      "id": 50,
+      "type": "CogVideoImageEncodeFunInP",
+      "pos": {
+        "0": 865,
+        "1": 567
+      },
+      "size": [
+        253.60000610351562,
+        146
+      ],
+      "flags": {},
+      "order": 13,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": 119
+        },
+        {
+          "name": "start_image",
+          "type": "IMAGE",
+          "link": 129
+        },
+        {
+          "name": "end_image",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "num_frames",
+          "type": "INT",
+          "link": 126,
+          "widget": {
+            "name": "num_frames"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "image_cond_latents",
+          "type": "LATENT",
+          "links": [
+            120
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoImageEncodeFunInP"
+      },
+      "widgets_values": [
+        33,
+        true,
+        0.03
+      ]
+    },
+    {
+      "id": 63,
+      "type": "CogVideoImageEncodeFunInP",
+      "pos": {
+        "0": 936.3893432617188,
+        "1": 1048.5242919921875
+      },
+      "size": {
+        "0": 253.60000610351562,
+        "1": 146
+      },
+      "flags": {},
+      "order": 11,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": 144
+        },
+        {
+          "name": "start_image",
+          "type": "IMAGE",
+          "link": 146
+        },
+        {
+          "name": "end_image",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "num_frames",
+          "type": "INT",
+          "link": 145,
+          "widget": {
+            "name": "num_frames"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "image_cond_latents",
+          "type": "LATENT",
+          "links": [
+            147
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoImageEncodeFunInP"
+      },
+      "widgets_values": [
+        33,
+        true,
+        0.03
+      ]
+    },
+    {
+      "id": 51,
+      "type": "CogVideoDecode",
+      "pos": {
+        "0": 1219,
+        "1": -134
+      },
+      "size": {
+        "0": 315,
+        "1": 198
+      },
+      "flags": {},
+      "order": 20,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": 122
+        },
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "link": 123
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            130
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoDecode"
+      },
+      "widgets_values": [
+        true,
+        240,
+        360,
+        0.2,
+        0.2,
+        true
+      ]
+    },
+    {
+      "id": 53,
+      "type": "PrimitiveNode",
+      "pos": {
+        "0": 117,
+        "1": 399
+      },
+      "size": [
+        261.57286031534534,
+        82
+      ],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "INT",
+          "type": "INT",
+          "links": [
+            126,
+            127,
+            143,
+            145
+          ],
+          "widget": {
+            "name": "num_frames"
+          },
+          "slot_index": 0
+        }
+      ],
+      "title": "num_frames",
+      "properties": {
+        "Run widget replace on values": false
+      },
+      "widgets_values": [
+        33,
+        "fixed"
+      ]
+    },
+    {
+      "id": 48,
+      "type": "CogVideoSampler",
+      "pos": {
+        "0": 1200,
+        "1": 124
+      },
+      "size": [
+        330,
+        574
+      ],
+      "flags": {},
+      "order": 18,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "model",
+          "type": "COGVIDEOMODEL",
+          "link": 114
+        },
+        {
+          "name": "positive",
+          "type": "CONDITIONING",
+          "link": 116
+        },
+        {
+          "name": "negative",
+          "type": "CONDITIONING",
+          "link": 117
+        },
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "image_cond_latents",
+          "type": "LATENT",
+          "link": 120,
+          "shape": 7
+        },
+        {
+          "name": "context_options",
+          "type": "COGCONTEXT",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "controlnet",
+          "type": "COGVIDECONTROLNET",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "tora_trajectory",
+          "type": "TORAFEATURES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "fastercache",
+          "type": "FASTERCACHEARGS",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "num_frames",
+          "type": "INT",
+          "link": 127,
+          "widget": {
+            "name": "num_frames"
+          }
+        },
+        {
+          "name": "seed",
+          "type": "INT",
+          "link": 156,
+          "widget": {
+            "name": "seed"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "links": [
+            123
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoSampler"
+      },
+      "widgets_values": [
+        33,
+        40,
+        6,
+        458091243358278,
+        "fixed",
+        "CogVideoXDDIM",
+        1
+      ]
+    },
+    {
+      "id": 68,
+      "type": "PrimitiveNode",
+      "pos": {
+        "0": 514,
+        "1": 985
+      },
+      "size": [
+        295.90419649751334,
+        82
+      ],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "INT",
+          "type": "INT",
+          "links": [
+            156,
+            157
+          ],
+          "widget": {
+            "name": "seed"
+          },
+          "slot_index": 0
+        }
+      ],
+      "title": "seed",
+      "properties": {
+        "Run widget replace on values": false
+      },
+      "widgets_values": [
+        458091243358278,
+        "fixed"
+      ]
+    },
+    {
+      "id": 69,
+      "type": "DownloadAndLoadFlorence2Model",
+      "pos": {
+        "0": -1305,
+        "1": -13
+      },
+      "size": [
+        442.37554309913344,
+        106
+      ],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "lora",
+          "type": "PEFTLORA",
+          "link": null,
+          "shape": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "florence2_model",
+          "type": "FL2MODEL",
+          "links": [
+            158
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DownloadAndLoadFlorence2Model"
+      },
+      "widgets_values": [
+        "MiaoshouAI/Florence-2-base-PromptGen-v2.0",
+        "fp16",
+        "sdpa"
+      ]
+    },
+    {
+      "id": 37,
+      "type": "ImageResizeKJ",
+      "pos": {
+        "0": -202,
+        "1": 588
+      },
+      "size": {
+        "0": 315,
+        "1": 266
+      },
+      "flags": {},
+      "order": 9,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 71
+        },
+        {
+          "name": "get_image_size",
+          "type": "IMAGE",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "width_input",
+          "type": "INT",
+          "link": null,
+          "widget": {
+            "name": "width_input"
+          }
+        },
+        {
+          "name": "height_input",
+          "type": "INT",
+          "link": null,
+          "widget": {
+            "name": "height_input"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            128,
+            146,
+            159
+          ],
+          "slot_index": 0,
+          "shape": 3
+        },
+        {
+          "name": "width",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        },
+        {
+          "name": "height",
+          "type": "INT",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ImageResizeKJ"
+      },
+      "widgets_values": [
+        768,
+        768,
+        "lanczos",
+        true,
+        2,
+        0,
+        0,
+        "disabled"
+      ]
+    },
+    {
+      "id": 71,
+      "type": "StringConstantMultiline",
+      "pos": {
+        "0": -709,
+        "1": 20
+      },
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            160
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "StringConstantMultiline"
+      },
+      "widgets_values": [
+        "camera orbit",
+        false
+      ]
+    },
+    {
+      "id": 72,
+      "type": "JoinStrings",
+      "pos": {
+        "0": -232,
+        "1": 231
+      },
+      "size": [
+        315,
+        106
+      ],
+      "flags": {},
+      "order": 15,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "string1",
+          "type": "STRING",
+          "link": 160,
+          "widget": {
+            "name": "string1"
+          }
+        },
+        {
+          "name": "string2",
+          "type": "STRING",
+          "link": 162,
+          "widget": {
+            "name": "string2"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": [
+            163
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "JoinStrings"
+      },
+      "widgets_values": [
+        "",
+        "",
+        ", "
+      ]
+    },
+    {
+      "id": 70,
+      "type": "Florence2Run",
+      "pos": {
+        "0": -1276,
+        "1": 170
+      },
+      "size": {
+        "0": 400,
+        "1": 352
+      },
+      "flags": {},
+      "order": 12,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 159
+        },
+        {
+          "name": "florence2_model",
+          "type": "FL2MODEL",
+          "link": 158
+        }
+      ],
+      "outputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "links": null,
+          "slot_index": 0
+        },
+        {
+          "name": "mask",
+          "type": "MASK",
+          "links": null
+        },
+        {
+          "name": "caption",
+          "type": "STRING",
+          "links": [
+            161,
+            162
+          ],
+          "slot_index": 2
+        },
+        {
+          "name": "data",
+          "type": "JSON",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "Florence2Run"
+      },
+      "widgets_values": [
+        "",
+        "more_detailed_caption",
+        true,
+        false,
+        226,
+        3,
+        true,
+        "",
+        586007018516875,
+        "fixed"
+      ]
+    },
+    {
+      "id": 73,
+      "type": "ShowText|pysssss",
+      "pos": {
+        "0": -793,
+        "1": 321
+      },
+      "size": [
+        502.3168660879171,
+        180.55015376950485
+      ],
+      "flags": {},
+      "order": 14,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 161,
+          "widget": {
+            "name": "text"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowText|pysssss"
+      },
+      "widgets_values": [
+        "",
+        "A digital illustration shoot from a frontal camera angle about a dark knight in shining armor stands in a dimly lit forest, with a glowing fire in the background. the image also shows a mysterious and intense atmosphere. on the middle of the image, a male knight appears to be standing, facing the viewer, with his full body visible. he is wearing a full plate armor with a red cloth draped over his shoulders. the armor is shiny and detailed, with intricate designs and a chain attached to it. he has two curved horns on his head, and his eyes are glowing yellow. the background is dark and smoky, with tall trees and a warm, glowing fire."
+      ]
+    },
+    {
+      "id": 59,
+      "type": "GIMMVFI_interpolate",
+      "pos": {
+        "0": 2880,
+        "1": -200
+      },
+      "size": {
+        "0": 330,
+        "1": 150
+      },
+      "flags": {},
+      "order": 27,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "gimmvfi_model",
+          "type": "GIMMVIF_MODEL",
+          "link": 134
+        },
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 165
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            164
+          ],
+          "slot_index": 0
+        },
+        {
+          "name": "flow_tensors",
+          "type": "IMAGE",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "GIMMVFI_interpolate"
+      },
+      "widgets_values": [
+        1,
+        2,
+        223874235763998,
+        "fixed"
+      ]
+    },
+    {
+      "id": 67,
+      "type": "ImageBatchMulti",
+      "pos": {
+        "0": 2900,
+        "1": 20
+      },
+      "size": {
+        "0": 210,
+        "1": 102
+      },
+      "flags": {},
+      "order": 26,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image_1",
+          "type": "IMAGE",
+          "link": 152
+        },
+        {
+          "name": "image_2",
+          "type": "IMAGE",
+          "link": 153
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            165
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {},
+      "widgets_values": [
+        2,
+        null
+      ]
+    },
+    {
+      "id": 66,
+      "type": "ReverseImageBatch",
+      "pos": {
+        "0": 2590,
+        "1": -20
+      },
+      "size": {
+        "0": 239.40000915527344,
+        "1": 26
+      },
+      "flags": {},
+      "order": 25,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 151
+        }
+      ],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            152
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ReverseImageBatch"
+      }
+    },
+    {
+      "id": 58,
+      "type": "DownloadAndLoadGIMMVFIModel",
+      "pos": {
+        "0": 2510,
+        "1": -210
+      },
+      "size": {
+        "0": 315,
+        "1": 58
+      },
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "gimmvfi_model",
+          "type": "GIMMVIF_MODEL",
+          "links": [
+            134
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DownloadAndLoadGIMMVFIModel"
+      },
+      "widgets_values": [
+        "gimmvfi_r_arb_lpips_fp32.safetensors"
+      ]
+    },
+    {
+      "id": 36,
+      "type": "LoadImage",
+      "pos": {
+        "0": -808,
+        "1": 573
+      },
+      "size": [
+        556.7343073028583,
+        502.50569947324857
+      ],
+      "flags": {},
+      "order": 7,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            71
+          ],
+          "slot_index": 0,
+          "shape": 3
+        },
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "ComfyUI_temp_lhgah_00059_.png",
+        "image"
+      ]
+    },
+    {
+      "id": 60,
+      "type": "VHS_VideoCombine",
+      "pos": {
+        "0": 2520,
+        "1": 180
+      },
+      "size": [
+        860.5738525390625,
+        1444.76513671875
+      ],
+      "flags": {},
+      "order": 28,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 164
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "meta_batch",
+          "type": "VHS_BatchManager",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": null,
+          "shape": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "Filenames",
+          "type": "VHS_FILENAMES",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VHS_VideoCombine"
+      },
+      "widgets_values": {
+        "frame_rate": 16,
+        "loop_count": 0,
+        "filename_prefix": "CogVideoX_Fun_orbits",
+        "format": "video/h264-mp4",
+        "pix_fmt": "yuv420p",
+        "crf": 19,
+        "save_metadata": true,
+        "pingpong": false,
+        "save_output": true,
+        "videopreview": {
+          "hidden": false,
+          "paused": false,
+          "params": {
+            "filename": "CogVideoX_Fun_orbits_00003.mp4",
+            "subfolder": "",
+            "type": "temp",
+            "format": "video/h264-mp4",
+            "frame_rate": 16
+          },
+          "muted": false
+        }
+      }
+    },
+    {
+      "id": 30,
+      "type": "CogVideoTextEncode",
+      "pos": {
+        "0": 478,
+        "1": 90
+      },
+      "size": [
+        471.90142822265625,
+        168.08047485351562
+      ],
+      "flags": {},
+      "order": 16,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "clip",
+          "type": "CLIP",
+          "link": 54
+        },
+        {
+          "name": "prompt",
+          "type": "STRING",
+          "link": 163,
+          "widget": {
+            "name": "prompt"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "conditioning",
+          "type": "CONDITIONING",
+          "links": [
+            116,
+            140
+          ],
+          "slot_index": 0,
+          "shape": 3
+        },
+        {
+          "name": "clip",
+          "type": "CLIP",
+          "links": [
+            110
+          ],
+          "slot_index": 1
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoTextEncode"
+      },
+      "widgets_values": [
+        "camera orbit around a mouse knight standing in a fantasy forest",
+        1,
+        false
+      ]
+    },
+    {
+      "id": 31,
+      "type": "CogVideoTextEncode",
+      "pos": {
+        "0": 493,
+        "1": 334
+      },
+      "size": {
+        "0": 463.01251220703125,
+        "1": 144
+      },
+      "flags": {},
+      "order": 17,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "clip",
+          "type": "CLIP",
+          "link": 110
+        }
+      ],
+      "outputs": [
+        {
+          "name": "conditioning",
+          "type": "CONDITIONING",
+          "links": [
+            117,
+            141
+          ],
+          "slot_index": 0,
+          "shape": 3
+        },
+        {
+          "name": "clip",
+          "type": "CLIP",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoTextEncode"
+      },
+      "widgets_values": [
+        "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. ",
+        1,
+        true
+      ]
+    },
+    {
+      "id": 62,
+      "type": "CogVideoSampler",
+      "pos": {
+        "0": 1258,
+        "1": 1151
+      },
+      "size": [
+        330,
+        574
+      ],
+      "flags": {},
+      "order": 19,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "model",
+          "type": "COGVIDEOMODEL",
+          "link": 139
+        },
+        {
+          "name": "positive",
+          "type": "CONDITIONING",
+          "link": 140
+        },
+        {
+          "name": "negative",
+          "type": "CONDITIONING",
+          "link": 141
+        },
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "image_cond_latents",
+          "type": "LATENT",
+          "link": 147,
+          "shape": 7
+        },
+        {
+          "name": "context_options",
+          "type": "COGCONTEXT",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "controlnet",
+          "type": "COGVIDECONTROLNET",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "tora_trajectory",
+          "type": "TORAFEATURES",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "fastercache",
+          "type": "FASTERCACHEARGS",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "num_frames",
+          "type": "INT",
+          "link": 143,
+          "widget": {
+            "name": "num_frames"
+          }
+        },
+        {
+          "name": "seed",
+          "type": "INT",
+          "link": 157,
+          "widget": {
+            "name": "seed"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "links": [
+            148
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoSampler"
+      },
+      "widgets_values": [
+        33,
+        40,
+        6,
+        458091243358278,
+        "fixed",
+        "CogVideoXDDIM",
+        1
+      ]
+    },
+    {
+      "id": 64,
+      "type": "CogVideoDecode",
+      "pos": {
+        "0": 1258,
+        "1": 889
+      },
+      "size": {
+        "0": 315,
+        "1": 198
+      },
+      "flags": {},
+      "order": 21,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": 149
+        },
+        {
+          "name": "samples",
+          "type": "LATENT",
+          "link": 148
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [
+            150,
+            153
+          ],
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "CogVideoDecode"
+      },
+      "widgets_values": [
+        true,
+        240,
+        360,
+        0.2,
+        0.2,
+        true
+      ]
+    },
+    {
+      "id": 44,
+      "type": "VHS_VideoCombine",
+      "pos": {
+        "0": 1652,
+        "1": -465
+      },
+      "size": [
+        592.7721081788095,
+        1087.6961669921875
+      ],
+      "flags": {},
+      "order": 24,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 131
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "meta_batch",
+          "type": "VHS_BatchManager",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": null,
+          "shape": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "Filenames",
+          "type": "VHS_FILENAMES",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VHS_VideoCombine"
+      },
+      "widgets_values": {
+        "frame_rate": 8,
+        "loop_count": 0,
+        "filename_prefix": "CogVideoX_Fun",
+        "format": "video/h264-mp4",
+        "pix_fmt": "yuv420p",
+        "crf": 19,
+        "save_metadata": true,
+        "pingpong": false,
+        "save_output": false,
+        "videopreview": {
+          "hidden": false,
+          "paused": false,
+          "params": {
+            "filename": "CogVideoX_Fun_00027.mp4",
+            "subfolder": "",
+            "type": "temp",
+            "format": "video/h264-mp4",
+            "frame_rate": 8
+          },
+          "muted": false
+        }
+      }
+    },
+    {
+      "id": 65,
+      "type": "VHS_VideoCombine",
+      "pos": {
+        "0": 1674,
+        "1": 688
+      },
+      "size": [
+        620.0130829180325,
+        1124.0174560546875
+      ],
+      "flags": {},
+      "order": 23,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 150
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "meta_batch",
+          "type": "VHS_BatchManager",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "vae",
+          "type": "VAE",
+          "link": null,
+          "shape": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "Filenames",
+          "type": "VHS_FILENAMES",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VHS_VideoCombine"
+      },
+      "widgets_values": {
+        "frame_rate": 8,
+        "loop_count": 0,
+        "filename_prefix": "CogVideoX_Fun",
+        "format": "video/h264-mp4",
+        "pix_fmt": "yuv420p",
+        "crf": 19,
+        "save_metadata": true,
+        "pingpong": false,
+        "save_output": false,
+        "videopreview": {
+          "hidden": false,
+          "paused": false,
+          "params": {
+            "filename": "CogVideoX_Fun_00026.mp4",
+            "subfolder": "",
+            "type": "temp",
+            "format": "video/h264-mp4",
+            "frame_rate": 8
+          },
+          "muted": false
+        }
+      }
+    },
+    {
+      "id": 49,
+      "type": "DownloadAndLoadCogVideoModel",
+      "pos": {
+        "0": 450,
+        "1": -217
+      },
+      "size": {
+        "0": 362.1656799316406,
+        "1": 218
+      },
+      "flags": {},
+      "order": 8,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "block_edit",
+          "type": "TRANSFORMERBLOCKS",
+          "link": null,
+          "shape": 7
+        },
+        {
+          "name": "lora",
+          "type": "COGLORA",
+          "link": 124,
+          "shape": 7
+        },
+        {
+          "name": "compile_args",
+          "type": "COMPILEARGS",
+          "link": null,
+          "shape": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "model",
+          "type": "COGVIDEOMODEL",
+          "links": [
+            114,
+            139
+          ]
+        },
+        {
+          "name": "vae",
+          "type": "VAE",
+          "links": [
+            119,
+            122,
+            144,
+            149
+          ],
+          "slot_index": 1
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "DownloadAndLoadCogVideoModel"
+      },
+      "widgets_values": [
+        "alibaba-pai/CogVideoX-Fun-V1.1-5b-InP",
+        "bf16",
+        "disabled",
+        false,
+        "sdpa",
+        "main_device"
+      ]
+    }
+  ],
+  "links": [
+    [
+      54,
+      20,
+      0,
+      30,
+      0,
+      "CLIP"
+    ],
+    [
+      71,
+      36,
+      0,
+      37,
+      0,
+      "IMAGE"
+    ],
+    [
+      110,
+      30,
+      1,
+      31,
+      0,
+      "CLIP"
+    ],
+    [
+      114,
+      49,
+      0,
+      48,
+      0,
+      "COGVIDEOMODEL"
+    ],
+    [
+      116,
+      30,
+      0,
+      48,
+      1,
+      "CONDITIONING"
+    ],
+    [
+      117,
+      31,
+      0,
+      48,
+      2,
+      "CONDITIONING"
+    ],
+    [
+      119,
+      49,
+      1,
+      50,
+      0,
+      "VAE"
+    ],
+    [
+      120,
+      50,
+      0,
+      48,
+      4,
+      "LATENT"
+    ],
+    [
+      122,
+      49,
+      1,
+      51,
+      0,
+      "VAE"
+    ],
+    [
+      123,
+      48,
+      0,
+      51,
+      1,
+      "LATENT"
+    ],
+    [
+      124,
+      52,
+      0,
+      49,
+      1,
+      "COGLORA"
+    ],
+    [
+      126,
+      53,
+      0,
+      50,
+      3,
+      "INT"
+    ],
+    [
+      127,
+      53,
+      0,
+      48,
+      9,
+      "INT"
+    ],
+    [
+      128,
+      37,
+      0,
+      54,
+      0,
+      "IMAGE"
+    ],
+    [
+      129,
+      54,
+      0,
+      50,
+      1,
+      "IMAGE"
+    ],
+    [
+      130,
+      51,
+      0,
+      55,
+      0,
+      "IMAGE"
+    ],
+    [
+      131,
+      55,
+      0,
+      44,
+      0,
+      "IMAGE"
+    ],
+    [
+      134,
+      58,
+      0,
+      59,
+      0,
+      "GIMMVIF_MODEL"
+    ],
+    [
+      139,
+      49,
+      0,
+      62,
+      0,
+      "COGVIDEOMODEL"
+    ],
+    [
+      140,
+      30,
+      0,
+      62,
+      1,
+      "CONDITIONING"
+    ],
+    [
+      141,
+      31,
+      0,
+      62,
+      2,
+      "CONDITIONING"
+    ],
+    [
+      143,
+      53,
+      0,
+      62,
+      9,
+      "INT"
+    ],
+    [
+      144,
+      49,
+      1,
+      63,
+      0,
+      "VAE"
+    ],
+    [
+      145,
+      53,
+      0,
+      63,
+      3,
+      "INT"
+    ],
+    [
+      146,
+      37,
+      0,
+      63,
+      1,
+      "IMAGE"
+    ],
+    [
+      147,
+      63,
+      0,
+      62,
+      4,
+      "LATENT"
+    ],
+    [
+      148,
+      62,
+      0,
+      64,
+      1,
+      "LATENT"
+    ],
+    [
+      149,
+      49,
+      1,
+      64,
+      0,
+      "VAE"
+    ],
+    [
+      150,
+      64,
+      0,
+      65,
+      0,
+      "IMAGE"
+    ],
+    [
+      151,
+      55,
+      0,
+      66,
+      0,
+      "IMAGE"
+    ],
+    [
+      152,
+      66,
+      0,
+      67,
+      0,
+      "IMAGE"
+    ],
+    [
+      153,
+      64,
+      0,
+      67,
+      1,
+      "IMAGE"
+    ],
+    [
+      156,
+      68,
+      0,
+      48,
+      10,
+      "INT"
+    ],
+    [
+      157,
+      68,
+      0,
+      62,
+      10,
+      "INT"
+    ],
+    [
+      158,
+      69,
+      0,
+      70,
+      1,
+      "FL2MODEL"
+    ],
+    [
+      159,
+      37,
+      0,
+      70,
+      0,
+      "IMAGE"
+    ],
+    [
+      160,
+      71,
+      0,
+      72,
+      0,
+      "STRING"
+    ],
+    [
+      161,
+      70,
+      2,
+      73,
+      0,
+      "STRING"
+    ],
+    [
+      162,
+      70,
+      2,
+      72,
+      1,
+      "STRING"
+    ],
+    [
+      163,
+      72,
+      0,
+      30,
+      1,
+      "STRING"
+    ],
+    [
+      164,
+      59,
+      0,
+      60,
+      0,
+      "IMAGE"
+    ],
+    [
+      165,
+      67,
+      0,
+      59,
+      1,
+      "IMAGE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 0.47362440744777223,
+      "offset": [
+        1633.9967545643788,
+        525.3824652843582
+      ]
+    }
+  },
+  "version": 0.4
+}

custom_nodes/ComfyUI-CogVideoXWrapper/model_loading.py

@@ -70,7 +70,6 @@ class CogVideoLoraSelect:
     RETURN_NAMES = ("lora", )
     FUNCTION = "getlorapath"
     CATEGORY = "CogVideoWrapper"
-    DESCRIPTION = "Select a LoRA model from ComfyUI/models/CogVideo/loras"
 
     def getlorapath(self, lora, strength, prev_lora=None, fuse_lora=False):
         cog_loras_list = []
@@ -87,43 +86,6 @@ class CogVideoLoraSelect:
         cog_loras_list.append(cog_lora)
         print(cog_loras_list)
         return (cog_loras_list,)
-
-class CogVideoLoraSelectComfy:
-    @classmethod
-    def INPUT_TYPES(s):
-        return {
-            "required": {
-               "lora": (folder_paths.get_filename_list("loras"), 
-                {"tooltip": "LORA models are expected to be in ComfyUI/models/loras with .safetensors extension"}),
-                "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.0001, "tooltip": "LORA strength, set to 0.0 to unmerge the LORA"}),
-            },
-            "optional": {
-                "prev_lora":("COGLORA", {"default": None, "tooltip": "For loading multiple LoRAs"}),
-                "fuse_lora": ("BOOLEAN", {"default": False, "tooltip": "Fuse the LoRA weights into the transformer"}),
-            }
-        }
-
-    RETURN_TYPES = ("COGLORA",)
-    RETURN_NAMES = ("lora", )
-    FUNCTION = "getlorapath"
-    CATEGORY = "CogVideoWrapper"
-    DESCRIPTION = "Select a LoRA model from ComfyUI/models/loras"
-
-    def getlorapath(self, lora, strength, prev_lora=None, fuse_lora=False):
-        cog_loras_list = []
-
-        cog_lora = {
-            "path": folder_paths.get_full_path("loras", lora),
-            "strength": strength,
-            "name": lora.split(".")[0],
-            "fuse_lora": fuse_lora
-        }
-        if prev_lora is not None:
-            cog_loras_list.extend(prev_lora)
-            
-        cog_loras_list.append(cog_lora)
-        print(cog_loras_list)
-        return (cog_loras_list,)
     
 #region DownloadAndLoadCogVideoModel
 class DownloadAndLoadCogVideoModel:
@@ -162,19 +124,7 @@ class DownloadAndLoadCogVideoModel:
                 "block_edit": ("TRANSFORMERBLOCKS", {"default": None}),
                 "lora": ("COGLORA", {"default": None}),
                 "compile_args":("COMPILEARGS", ),
-                "attention_mode": ([
-                    "sdpa",
-                    "fused_sdpa",
-                    "sageattn",
-                    "fused_sageattn", 
-                    "sageattn_qk_int8_pv_fp8_cuda",
-                    "sageattn_qk_int8_pv_fp16_cuda",
-                    "sageattn_qk_int8_pv_fp16_triton",
-                    "fused_sageattn_qk_int8_pv_fp8_cuda",
-                    "fused_sageattn_qk_int8_pv_fp16_cuda",
-                    "fused_sageattn_qk_int8_pv_fp16_triton",
-                    "comfy"
-                    ], {"default": "sdpa"}),
+                "attention_mode": (["sdpa", "sageattn", "fused_sdpa", "fused_sageattn", "comfy"], {"default": "sdpa"}),
                 "load_device": (["main_device", "offload_device"], {"default": "main_device"}),
             }
         }
@@ -189,18 +139,11 @@ class DownloadAndLoadCogVideoModel:
                   enable_sequential_cpu_offload=False, block_edit=None, lora=None, compile_args=None, 
                   attention_mode="sdpa", load_device="main_device"):
         
-        transformer = None
-
         if "sage" in attention_mode:
             try:
                 from sageattention import sageattn
             except Exception as e:
                 raise ValueError(f"Can't import SageAttention: {str(e)}")
-            if "qk_int8" in attention_mode:
-                try:
-                    from sageattention import sageattn_qk_int8_pv_fp16_cuda
-                except Exception as e:
-                    raise ValueError(f"Can't import SageAttention 2.0.0: {str(e)}")
         
         if precision == "fp16" and "1.5" in model:
             raise ValueError("1.5 models do not currently work in fp16")
@@ -275,7 +218,7 @@ class DownloadAndLoadCogVideoModel:
                 local_dir_use_symlinks=False,
             )
 
-        transformer = CogVideoXTransformer3DModel.from_pretrained(base_path, subfolder=subfolder, attention_mode=attention_mode)
+        transformer = CogVideoXTransformer3DModel.from_pretrained(base_path, subfolder=subfolder)
         transformer = transformer.to(dtype).to(transformer_load_device)
 
         if "1.5" in model:
@@ -348,6 +291,7 @@ class DownloadAndLoadCogVideoModel:
             for module in pipe.transformer.modules():
                 if isinstance(module, Attention):
                     module.fuse_projections(fuse=True)
+        pipe.transformer.attention_mode = attention_mode
 
         if compile_args is not None:
             pipe.transformer.to(memory_format=torch.channels_last)
@@ -571,7 +515,7 @@ class DownloadAndLoadCogVideoGGUFModel:
             else:
                 transformer_config["in_channels"] = 16
 
-            transformer = CogVideoXTransformer3DModel.from_config(transformer_config, attention_mode=attention_mode)
+            transformer = CogVideoXTransformer3DModel.from_config(transformer_config)
             cast_dtype = vae_dtype
             params_to_keep = {"patch_embed", "pos_embedding", "time_embedding"}
             if "2b" in model:
@@ -675,19 +619,7 @@ class CogVideoXModelLoader:
                 "block_edit": ("TRANSFORMERBLOCKS", {"default": None}),
                 "lora": ("COGLORA", {"default": None}),
                 "compile_args":("COMPILEARGS", ),
-                "attention_mode": ([
-                    "sdpa",
-                    "fused_sdpa",
-                    "sageattn",
-                    "fused_sageattn", 
-                    "sageattn_qk_int8_pv_fp8_cuda",
-                    "sageattn_qk_int8_pv_fp16_cuda",
-                    "sageattn_qk_int8_pv_fp16_triton",
-                    "fused_sageattn_qk_int8_pv_fp8_cuda",
-                    "fused_sageattn_qk_int8_pv_fp16_cuda",
-                    "fused_sageattn_qk_int8_pv_fp16_triton",
-                    "comfy"
-                    ], {"default": "sdpa"}),
+                "attention_mode": (["sdpa", "sageattn", "fused_sdpa", "fused_sageattn"], {"default": "sdpa"}),
             }
         }
 
@@ -698,7 +630,7 @@ class CogVideoXModelLoader:
 
     def loadmodel(self, model, base_precision, load_device, enable_sequential_cpu_offload, 
                   block_edit=None, compile_args=None, lora=None, attention_mode="sdpa", quantization="disabled"):
-        transformer = None
+        
         if "sage" in attention_mode:
             try:
                 from sageattention import sageattn
@@ -764,7 +696,7 @@ class CogVideoXModelLoader:
                     transformer_config["sample_width"] = 300
 
         with init_empty_weights():
-            transformer = CogVideoXTransformer3DModel.from_config(transformer_config, attention_mode=attention_mode)
+            transformer = CogVideoXTransformer3DModel.from_config(transformer_config)
 
         #load weights
         #params_to_keep = {}
@@ -1116,7 +1048,6 @@ NODE_CLASS_MAPPINGS = {
     "CogVideoLoraSelect": CogVideoLoraSelect,
     "CogVideoXVAELoader": CogVideoXVAELoader,
     "CogVideoXModelLoader": CogVideoXModelLoader,
-    "CogVideoLoraSelectComfy": CogVideoLoraSelectComfy
 }
 NODE_DISPLAY_NAME_MAPPINGS = {
     "DownloadAndLoadCogVideoModel": "(Down)load CogVideo Model",
@@ -1126,5 +1057,4 @@ NODE_DISPLAY_NAME_MAPPINGS = {
     "CogVideoLoraSelect": "CogVideo LoraSelect",
     "CogVideoXVAELoader": "CogVideoX VAE Loader",
     "CogVideoXModelLoader": "CogVideoX Model Loader",
-    "CogVideoLoraSelectComfy": "CogVideo LoraSelect Comfy"
     }

custom_nodes/ComfyUI-DepthAnythingV2/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

custom_nodes/ComfyUI-DepthAnythingV2/.github/workflows/publish.yml

@@ -0,0 +1,22 @@
+name: Publish to Comfy registry
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - master
+    paths:
+      - "pyproject.toml"
+
+jobs:
+  publish-node:
+    name: Publish Custom Node to registry
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Publish Custom Node
+        uses: Comfy-Org/publish-node-action@main
+        with:
+          ## Add your own personal access token to your Github Repository secrets and reference it here.
+          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

custom_nodes/ComfyUI-DepthAnythingV2/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+*pyc
+.vscode
+__pycache__
+*.egg-info
+*.bak
+checkpoints
+results
+backup

custom_nodes/ComfyUI-DepthAnythingV2/README.md

@@ -0,0 +1,6 @@
+# ComfyUI nodes to use DepthAnythingV2
+https://depth-anything-v2.github.io
+
+Models autodownload to `ComfyUI\models\depthanything` from https://huggingface.co/Kijai/DepthAnythingV2-safetensors/tree/main
+
+![image](https://github.com/kijai/ComfyUI-DepthAnythingV2/assets/40791699/6e9f190a-02da-4176-9cca-6e4e4b0ad35d)

custom_nodes/ComfyUI-DepthAnythingV2/__init__.py

@@ -0,0 +1,3 @@
+from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
+
+__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2.py

@@ -0,0 +1,415 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+from functools import partial
+import math
+import logging
+from typing import Sequence, Tuple, Union, Callable
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from torch.nn.init import trunc_normal_
+
+from .dinov2_layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block
+
+
+logger = logging.getLogger("dinov2")
+
+
+def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
+    if not depth_first and include_root:
+        fn(module=module, name=name)
+    for child_name, child_module in module.named_children():
+        child_name = ".".join((name, child_name)) if name else child_name
+        named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True)
+    if depth_first and include_root:
+        fn(module=module, name=name)
+    return module
+
+
+class BlockChunk(nn.ModuleList):
+    def forward(self, x):
+        for b in self:
+            x = b(x)
+        return x
+
+
+class DinoVisionTransformer(nn.Module):
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=16,
+        in_chans=3,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4.0,
+        qkv_bias=True,
+        ffn_bias=True,
+        proj_bias=True,
+        drop_path_rate=0.0,
+        drop_path_uniform=False,
+        init_values=None,  # for layerscale: None or 0 => no layerscale
+        embed_layer=PatchEmbed,
+        act_layer=nn.GELU,
+        block_fn=Block,
+        ffn_layer="mlp",
+        block_chunks=1,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+    ):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_chans (int): number of input channels
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            proj_bias (bool): enable bias for proj in attn if True
+            ffn_bias (bool): enable bias for ffn if True
+            drop_path_rate (float): stochastic depth rate
+            drop_path_uniform (bool): apply uniform drop rate across blocks
+            weight_init (str): weight init scheme
+            init_values (float): layer-scale init values
+            embed_layer (nn.Module): patch embedding layer
+            act_layer (nn.Module): MLP activation layer
+            block_fn (nn.Module): transformer block class
+            ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
+            block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
+            num_register_tokens: (int) number of extra cls tokens (so-called "registers")
+            interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings
+            interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings
+        """
+        super().__init__()
+        norm_layer = partial(nn.LayerNorm, eps=1e-6)
+
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        self.num_tokens = 1
+        self.n_blocks = depth
+        self.num_heads = num_heads
+        self.patch_size = patch_size
+        self.num_register_tokens = num_register_tokens
+        self.interpolate_antialias = interpolate_antialias
+        self.interpolate_offset = interpolate_offset
+
+        self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
+        assert num_register_tokens >= 0
+        self.register_tokens = (
+            nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None
+        )
+
+        if drop_path_uniform is True:
+            dpr = [drop_path_rate] * depth
+        else:
+            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+
+        if ffn_layer == "mlp":
+            logger.info("using MLP layer as FFN")
+            ffn_layer = Mlp
+        elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
+            logger.info("using SwiGLU layer as FFN")
+            ffn_layer = SwiGLUFFNFused
+        elif ffn_layer == "identity":
+            logger.info("using Identity layer as FFN")
+
+            def f(*args, **kwargs):
+                return nn.Identity()
+
+            ffn_layer = f
+        else:
+            raise NotImplementedError
+
+        blocks_list = [
+            block_fn(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                proj_bias=proj_bias,
+                ffn_bias=ffn_bias,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                ffn_layer=ffn_layer,
+                init_values=init_values,
+            )
+            for i in range(depth)
+        ]
+        if block_chunks > 0:
+            self.chunked_blocks = True
+            chunked_blocks = []
+            chunksize = depth // block_chunks
+            for i in range(0, depth, chunksize):
+                # this is to keep the block index consistent if we chunk the block list
+                chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize])
+            self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
+        else:
+            self.chunked_blocks = False
+            self.blocks = nn.ModuleList(blocks_list)
+
+        self.norm = norm_layer(embed_dim)
+        self.head = nn.Identity()
+
+        self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
+
+        self.init_weights()
+
+    def init_weights(self):
+        trunc_normal_(self.pos_embed, std=0.02)
+        nn.init.normal_(self.cls_token, std=1e-6)
+        if self.register_tokens is not None:
+            nn.init.normal_(self.register_tokens, std=1e-6)
+        named_apply(init_weights_vit_timm, self)
+
+    def interpolate_pos_encoding(self, x, w, h):
+        previous_dtype = x.dtype
+        npatch = x.shape[1] - 1
+        N = self.pos_embed.shape[1] - 1
+        if npatch == N and w == h:
+            return self.pos_embed
+        pos_embed = self.pos_embed.float()
+        class_pos_embed = pos_embed[:, 0]
+        patch_pos_embed = pos_embed[:, 1:]
+        dim = x.shape[-1]
+        w0 = w // self.patch_size
+        h0 = h // self.patch_size
+        # we add a small number to avoid floating point error in the interpolation
+        # see discussion at https://github.com/facebookresearch/dino/issues/8
+        # DINOv2 with register modify the interpolate_offset from 0.1 to 0.0
+        w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset
+        # w0, h0 = w0 + 0.1, h0 + 0.1
+        
+        sqrt_N = math.sqrt(N)
+        sx, sy = float(w0) / sqrt_N, float(h0) / sqrt_N
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed.reshape(1, int(sqrt_N), int(sqrt_N), dim).permute(0, 3, 1, 2),
+            scale_factor=(sx, sy),
+            # (int(w0), int(h0)), # to solve the upsampling shape issue
+            mode="bicubic",
+            antialias=self.interpolate_antialias
+        )
+        
+        assert int(w0) == patch_pos_embed.shape[-2]
+        assert int(h0) == patch_pos_embed.shape[-1]
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype)
+
+    def prepare_tokens_with_masks(self, x, masks=None):
+        B, nc, w, h = x.shape
+        x = self.patch_embed(x)
+        if masks is not None:
+            x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x)
+
+        x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+        x = x + self.interpolate_pos_encoding(x, w, h)
+
+        if self.register_tokens is not None:
+            x = torch.cat(
+                (
+                    x[:, :1],
+                    self.register_tokens.expand(x.shape[0], -1, -1),
+                    x[:, 1:],
+                ),
+                dim=1,
+            )
+
+        return x
+
+    def forward_features_list(self, x_list, masks_list):
+        x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)]
+        for blk in self.blocks:
+            x = blk(x)
+
+        all_x = x
+        output = []
+        for x, masks in zip(all_x, masks_list):
+            x_norm = self.norm(x)
+            output.append(
+                {
+                    "x_norm_clstoken": x_norm[:, 0],
+                    "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
+                    "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+                    "x_prenorm": x,
+                    "masks": masks,
+                }
+            )
+        return output
+
+    def forward_features(self, x, masks=None):
+        if isinstance(x, list):
+            return self.forward_features_list(x, masks)
+
+        x = self.prepare_tokens_with_masks(x, masks)
+
+        for blk in self.blocks:
+            x = blk(x)
+
+        x_norm = self.norm(x)
+        return {
+            "x_norm_clstoken": x_norm[:, 0],
+            "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
+            "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+            "x_prenorm": x,
+            "masks": masks,
+        }
+
+    def _get_intermediate_layers_not_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        # If n is an int, take the n last blocks. If it's a list, take them
+        output, total_block_len = [], len(self.blocks)
+        blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+        for i, blk in enumerate(self.blocks):
+            x = blk(x)
+            if i in blocks_to_take:
+                output.append(x)
+        assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+        return output
+
+    def _get_intermediate_layers_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        output, i, total_block_len = [], 0, len(self.blocks[-1])
+        # If n is an int, take the n last blocks. If it's a list, take them
+        blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+        for block_chunk in self.blocks:
+            for blk in block_chunk[i:]:  # Passing the nn.Identity()
+                x = blk(x)
+                if i in blocks_to_take:
+                    output.append(x)
+                i += 1
+        assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+        return output
+
+    def get_intermediate_layers(
+        self,
+        x: torch.Tensor,
+        n: Union[int, Sequence] = 1,  # Layers or n last layers to take
+        reshape: bool = False,
+        return_class_token: bool = False,
+        norm=True
+    ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
+        if self.chunked_blocks:
+            outputs = self._get_intermediate_layers_chunked(x, n)
+        else:
+            outputs = self._get_intermediate_layers_not_chunked(x, n)
+        if norm:
+            outputs = [self.norm(out) for out in outputs]
+        class_tokens = [out[:, 0] for out in outputs]
+        outputs = [out[:, 1 + self.num_register_tokens:] for out in outputs]
+        if reshape:
+            B, _, w, h = x.shape
+            outputs = [
+                out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
+                for out in outputs
+            ]
+        if return_class_token:
+            return tuple(zip(outputs, class_tokens))
+        return tuple(outputs)
+
+    def forward(self, *args, is_training=False, **kwargs):
+        ret = self.forward_features(*args, **kwargs)
+        if is_training:
+            return ret
+        else:
+            return self.head(ret["x_norm_clstoken"])
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ""):
+    """ViT weight initialization, original timm impl (for reproducibility)"""
+    if isinstance(module, nn.Linear):
+        trunc_normal_(module.weight, std=0.02)
+        if module.bias is not None:
+            nn.init.zeros_(module.bias)
+
+
+def vit_small(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=384,
+        depth=12,
+        num_heads=6,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_base(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_large(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs):
+    """
+    Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64
+    """
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1536,
+        depth=40,
+        num_heads=24,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def DINOv2(model_name):
+    model_zoo = {
+        "vits": vit_small, 
+        "vitb": vit_base, 
+        "vitl": vit_large, 
+        "vitg": vit_giant2
+    }
+    
+    return model_zoo[model_name](
+        img_size=518,
+        patch_size=14,
+        init_values=1.0,
+        ffn_layer="mlp" if model_name != "vitg" else "swiglufused",
+        block_chunks=0,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1
+    )

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
+from .block import NestedTensorBlock
+from .attention import MemEffAttention

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/attention.py

@@ -0,0 +1,94 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+
+from torch import Tensor
+from torch import nn
+import comfy.ops
+ops = comfy.ops.manual_cast
+from comfy.ldm.modules.attention import optimized_attention
+
+logger = logging.getLogger("dinov2")
+
+try:
+    from xformers.ops import memory_efficient_attention, unbind
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    logger.warning("xFormers not available")
+    XFORMERS_AVAILABLE = False
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.scale = self.head_dim**-0.5
+
+        self.qkv = ops.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = ops.Linear(dim, dim, bias=proj_bias)
+        self.proj_drop = nn.Dropout(proj_drop)
+        
+
+    def forward(self, x: Tensor) -> Tensor:
+        # B, N, C = x.shape
+        # qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+
+        # q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+        # attn = q @ k.transpose(-2, -1)
+
+        # attn = attn.softmax(dim=-1)
+        # #attn = self.attn_drop(attn)
+
+        # x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        # x = self.proj(x)
+        # #x = self.proj_drop(x)
+        # return x
+        # print("x shape: ", x.shape)
+       
+        B, N, C = x.shape
+        q, k, v = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        out = optimized_attention(q, k, v, self.num_heads, skip_reshape=True)
+
+        out= self.proj(out)
+        out = self.proj_drop(out)
+        return out
+        
+
+class MemEffAttention(Attention):
+    def forward(self, x: Tensor, attn_bias=None) -> Tensor:
+        if not XFORMERS_AVAILABLE:
+            assert attn_bias is None, "xFormers is required for nested tensors usage"
+            return super().forward(x)
+
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
+
+        q, k, v = unbind(qkv, 2)
+
+        x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+        x = x.reshape([B, N, C])
+
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+        

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/block.py

@@ -0,0 +1,252 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+import logging
+from typing import Callable, List, Any, Tuple, Dict
+
+import torch
+from torch import nn, Tensor
+
+from .attention import Attention, MemEffAttention
+from .drop_path import DropPath
+from .layer_scale import LayerScale
+from .mlp import Mlp
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+    from xformers.ops import fmha
+    from xformers.ops import scaled_index_add, index_select_cat
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    logger.warning("xFormers not available")
+    XFORMERS_AVAILABLE = False
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        ffn_bias: bool = True,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+        attn_class: Callable[..., nn.Module] = Attention,
+        ffn_layer: Callable[..., nn.Module] = Mlp,
+    ) -> None:
+        super().__init__()
+        # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
+        self.norm1 = norm_layer(dim)
+        self.attn = attn_class(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+        )
+        self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = ffn_layer(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            bias=ffn_bias,
+        )
+        self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.sample_drop_ratio = drop_path
+
+    def forward(self, x: Tensor) -> Tensor:
+        def attn_residual_func(x: Tensor) -> Tensor:
+            return self.ls1(self.attn(self.norm1(x)))
+
+        def ffn_residual_func(x: Tensor) -> Tensor:
+            return self.ls2(self.mlp(self.norm2(x)))
+
+        if self.training and self.sample_drop_ratio > 0.1:
+            # the overhead is compensated only for a drop path rate larger than 0.1
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+        elif self.training and self.sample_drop_ratio > 0.0:
+            x = x + self.drop_path1(attn_residual_func(x))
+            x = x + self.drop_path1(ffn_residual_func(x))  # FIXME: drop_path2
+        else:
+            x = x + attn_residual_func(x)
+            x = x + ffn_residual_func(x)
+        return x
+
+
+def drop_add_residual_stochastic_depth(
+    x: Tensor,
+    residual_func: Callable[[Tensor], Tensor],
+    sample_drop_ratio: float = 0.0,
+) -> Tensor:
+    # 1) extract subset using permutation
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    x_subset = x[brange]
+
+    # 2) apply residual_func to get residual
+    residual = residual_func(x_subset)
+
+    x_flat = x.flatten(1)
+    residual = residual.flatten(1)
+
+    residual_scale_factor = b / sample_subset_size
+
+    # 3) add the residual
+    x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+    return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    residual_scale_factor = b / sample_subset_size
+    return brange, residual_scale_factor
+
+
+def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None):
+    if scaling_vector is None:
+        x_flat = x.flatten(1)
+        residual = residual.flatten(1)
+        x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+    else:
+        x_plus_residual = scaled_index_add(
+            x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor
+        )
+    return x_plus_residual
+
+
+attn_bias_cache: Dict[Tuple, Any] = {}
+
+
+def get_attn_bias_and_cat(x_list, branges=None):
+    """
+    this will perform the index select, cat the tensors, and provide the attn_bias from cache
+    """
+    batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list]
+    all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
+    if all_shapes not in attn_bias_cache.keys():
+        seqlens = []
+        for b, x in zip(batch_sizes, x_list):
+            for _ in range(b):
+                seqlens.append(x.shape[1])
+        attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
+        attn_bias._batch_sizes = batch_sizes
+        attn_bias_cache[all_shapes] = attn_bias
+
+    if branges is not None:
+        cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1])
+    else:
+        tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
+        cat_tensors = torch.cat(tensors_bs1, dim=1)
+
+    return attn_bias_cache[all_shapes], cat_tensors
+
+
+def drop_add_residual_stochastic_depth_list(
+    x_list: List[Tensor],
+    residual_func: Callable[[Tensor, Any], Tensor],
+    sample_drop_ratio: float = 0.0,
+    scaling_vector=None,
+) -> Tensor:
+    # 1) generate random set of indices for dropping samples in the batch
+    branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list]
+    branges = [s[0] for s in branges_scales]
+    residual_scale_factors = [s[1] for s in branges_scales]
+
+    # 2) get attention bias and index+concat the tensors
+    attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
+
+    # 3) apply residual_func to get residual, and split the result
+    residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias))  # type: ignore
+
+    outputs = []
+    for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors):
+        outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x))
+    return outputs
+
+
+class NestedTensorBlock(Block):
+    def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
+        """
+        x_list contains a list of tensors to nest together and run
+        """
+        assert isinstance(self.attn, MemEffAttention)
+
+        if self.training and self.sample_drop_ratio > 0.0:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.attn(self.norm1(x), attn_bias=attn_bias)
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.mlp(self.norm2(x))
+
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None,
+            )
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None,
+            )
+            return x_list
+        else:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls2(self.mlp(self.norm2(x)))
+
+            attn_bias, x = get_attn_bias_and_cat(x_list)
+            x = x + attn_residual_func(x, attn_bias=attn_bias)
+            x = x + ffn_residual_func(x)
+            return attn_bias.split(x)
+
+    def forward(self, x_or_x_list):
+        if isinstance(x_or_x_list, Tensor):
+            return super().forward(x_or_x_list)
+        elif isinstance(x_or_x_list, list):
+            assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage"
+            return self.forward_nested(x_or_x_list)
+        else:
+            raise AssertionError

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/drop_path.py

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
+
+
+from torch import nn
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False):
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0:
+        random_tensor.div_(keep_prob)
+    output = x * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
+
+    def __init__(self, drop_prob=None):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/layer_scale.py

@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110
+
+from typing import Union
+
+import torch
+from torch import Tensor
+from torch import nn
+
+
+class LayerScale(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        init_values: Union[float, Tensor] = 1e-5,
+        inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/mlp.py

@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
+
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/patch_embed.py

@@ -0,0 +1,90 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+from typing import Callable, Optional, Tuple, Union
+
+from torch import Tensor
+import torch.nn as nn
+import comfy.ops
+ops = comfy.ops.manual_cast
+
+def make_2tuple(x):
+    if isinstance(x, tuple):
+        assert len(x) == 2
+        return x
+
+    assert isinstance(x, int)
+    return (x, x)
+
+
+class PatchEmbed(nn.Module):
+    """
+    2D image to patch embedding: (B,C,H,W) -> (B,N,D)
+
+    Args:
+        img_size: Image size.
+        patch_size: Patch token size.
+        in_chans: Number of input image channels.
+        embed_dim: Number of linear projection output channels.
+        norm_layer: Normalization layer.
+    """
+
+    def __init__(
+        self,
+        img_size: Union[int, Tuple[int, int]] = 224,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+        norm_layer: Optional[Callable] = None,
+        flatten_embedding: bool = True,
+    ) -> None:
+        super().__init__()
+
+        image_HW = make_2tuple(img_size)
+        patch_HW = make_2tuple(patch_size)
+        patch_grid_size = (
+            image_HW[0] // patch_HW[0],
+            image_HW[1] // patch_HW[1],
+        )
+
+        self.img_size = image_HW
+        self.patch_size = patch_HW
+        self.patches_resolution = patch_grid_size
+        self.num_patches = patch_grid_size[0] * patch_grid_size[1]
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.flatten_embedding = flatten_embedding
+
+        self.proj = ops.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW)
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x: Tensor) -> Tensor:
+        _, _, H, W = x.shape
+        patch_H, patch_W = self.patch_size
+
+        assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}"
+        assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}"
+
+        x = self.proj(x)  # B C H W
+        H, W = x.size(2), x.size(3)
+        x = x.flatten(2).transpose(1, 2)  # B HW C
+        x = self.norm(x)
+        if not self.flatten_embedding:
+            x = x.reshape(-1, H, W, self.embed_dim)  # B H W C
+        return x
+
+    def flops(self) -> float:
+        Ho, Wo = self.patches_resolution
+        flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dinov2_layers/swiglu_ffn.py

@@ -0,0 +1,63 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+import torch.nn.functional as F
+
+
+class SwiGLUFFN(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
+        self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x12 = self.w12(x)
+        x1, x2 = x12.chunk(2, dim=-1)
+        hidden = F.silu(x1) * x2
+        return self.w3(hidden)
+
+
+try:
+    from xformers.ops import SwiGLU
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    SwiGLU = SwiGLUFFN
+    XFORMERS_AVAILABLE = False
+
+
+class SwiGLUFFNFused(SwiGLU):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
+        super().__init__(
+            in_features=in_features,
+            hidden_features=hidden_features,
+            out_features=out_features,
+            bias=bias,
+        )

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/dpt.py

@@ -0,0 +1,199 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .dinov2 import DINOv2
+from .util.blocks import FeatureFusionBlock, _make_scratch
+
+import comfy.ops
+ops = comfy.ops.manual_cast
+
+def _make_fusion_block(features, use_bn, size=None):
+    return FeatureFusionBlock(
+        features,
+        nn.ReLU(False),
+        deconv=False,
+        bn=use_bn,
+        expand=False,
+        align_corners=True,
+        size=size,
+    )
+
+class ConvBlock(nn.Module):
+    def __init__(self, in_feature, out_feature):
+        super().__init__()
+        
+        self.conv_block = nn.Sequential(
+            ops.Conv2d(in_feature, out_feature, kernel_size=3, stride=1, padding=1),
+            nn.BatchNorm2d(out_feature),
+            nn.ReLU(True)
+        )
+    
+    def forward(self, x):
+        return self.conv_block(x)
+
+class DPTHead(nn.Module):
+    def __init__(
+        self, 
+        in_channels, 
+        features=256, 
+        use_bn=False, 
+        out_channels=[256, 512, 1024, 1024], 
+        use_clstoken=False,
+        is_metric=False
+    ):
+        super(DPTHead, self).__init__()
+        
+        self.use_clstoken = use_clstoken
+        self.is_metric=is_metric
+        
+        self.projects = nn.ModuleList([
+            ops.Conv2d(
+                in_channels=in_channels,
+                out_channels=out_channel,
+                kernel_size=1,
+                stride=1,
+                padding=0,
+            ) for out_channel in out_channels
+        ])
+        
+        self.resize_layers = nn.ModuleList([
+            nn.ConvTranspose2d(
+                in_channels=out_channels[0],
+                out_channels=out_channels[0],
+                kernel_size=4,
+                stride=4,
+                padding=0),
+            nn.ConvTranspose2d(
+                in_channels=out_channels[1],
+                out_channels=out_channels[1],
+                kernel_size=2,
+                stride=2,
+                padding=0),
+            nn.Identity(),
+            ops.Conv2d(
+                in_channels=out_channels[3],
+                out_channels=out_channels[3],
+                kernel_size=3,
+                stride=2,
+                padding=1)
+        ])
+        
+        if use_clstoken:
+            self.readout_projects = nn.ModuleList()
+            for _ in range(len(self.projects)):
+                self.readout_projects.append(
+                    nn.Sequential(
+                        ops.Linear(2 * in_channels, in_channels),
+                        nn.GELU()))
+        
+        self.scratch = _make_scratch(
+            out_channels,
+            features,
+            groups=1,
+            expand=False,
+        )
+        
+        self.scratch.stem_transpose = None
+        
+        self.scratch.refinenet1 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet2 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet3 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet4 = _make_fusion_block(features, use_bn)
+        
+        head_features_1 = features
+        head_features_2 = 32
+        
+        self.scratch.output_conv1 = ops.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1)
+        if self.is_metric:
+            self.scratch.output_conv2 = nn.Sequential(
+                ops.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(True),
+                ops.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0),
+                nn.Sigmoid()
+            )
+        else:
+            self.scratch.output_conv2 = nn.Sequential(
+                ops.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(True),
+                ops.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0),
+                nn.ReLU(True),
+                nn.Identity(),
+            )
+    
+    def forward(self, out_features, patch_h, patch_w):
+        out = []
+        for i, x in enumerate(out_features):
+            if self.use_clstoken:
+                x, cls_token = x[0], x[1]
+                readout = cls_token.unsqueeze(1).expand_as(x)
+                x = self.readout_projects[i](torch.cat((x, readout), -1))
+            else:
+                x = x[0]
+            
+            x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w))
+            
+            x = self.projects[i](x)
+            x = self.resize_layers[i](x)
+            
+            out.append(x)
+        
+        layer_1, layer_2, layer_3, layer_4 = out
+        
+        layer_1_rn = self.scratch.layer1_rn(layer_1)
+        layer_2_rn = self.scratch.layer2_rn(layer_2)
+        layer_3_rn = self.scratch.layer3_rn(layer_3)
+        layer_4_rn = self.scratch.layer4_rn(layer_4)
+        
+        path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])        
+        path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:])
+        path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:])
+        path_1 = self.scratch.refinenet1(path_2, layer_1_rn)
+        
+        out = self.scratch.output_conv1(path_1)
+        out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True)
+        out = self.scratch.output_conv2(out)
+        
+        return out
+
+
+class DepthAnythingV2(nn.Module):
+    def __init__(
+        self, 
+        encoder='vitl', 
+        features=256, 
+        out_channels=[256, 512, 1024, 1024], 
+        use_bn=False, 
+        use_clstoken=False,
+        is_metric=False,
+        max_depth=20.0
+    ):
+        super(DepthAnythingV2, self).__init__()
+        
+        self.intermediate_layer_idx = {
+            'vits': [2, 5, 8, 11],
+            'vitb': [2, 5, 8, 11], 
+            'vitl': [4, 11, 17, 23], 
+            'vitg': [9, 19, 29, 39]
+        }
+        
+        self.is_metric = is_metric
+        self.max_depth = max_depth
+        
+        self.encoder = encoder
+        self.pretrained = DINOv2(model_name=encoder)
+        
+        self.depth_head = DPTHead(self.pretrained.embed_dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken, is_metric=is_metric)
+    
+    def forward(self, x):
+        patch_h, patch_w = x.shape[-2] // 14, x.shape[-1] // 14
+        
+        features = self.pretrained.get_intermediate_layers(x, self.intermediate_layer_idx[self.encoder], return_class_token=True)
+        
+        if self.is_metric:
+            depth = self.depth_head(features, patch_h, patch_w) * self.max_depth
+        else:
+            depth = self.depth_head(features, patch_h, patch_w)
+            depth = F.relu(depth)
+        
+        return depth.squeeze(1)

custom_nodes/ComfyUI-DepthAnythingV2/depth_anything_v2/util/blocks.py

@@ -0,0 +1,149 @@
+import torch.nn as nn
+import comfy.ops
+ops = comfy.ops.manual_cast
+
+def _make_scratch(in_shape, out_shape, groups=1, expand=False):
+    scratch = nn.Module()
+
+    out_shape1 = out_shape
+    out_shape2 = out_shape
+    out_shape3 = out_shape
+    if len(in_shape) >= 4:
+        out_shape4 = out_shape
+
+    if expand:
+        out_shape1 = out_shape
+        out_shape2 = out_shape * 2
+        out_shape3 = out_shape * 4
+        if len(in_shape) >= 4:
+            out_shape4 = out_shape * 8
+
+    scratch.layer1_rn = ops.Conv2d(in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+    scratch.layer2_rn = ops.Conv2d(in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+    scratch.layer3_rn = ops.Conv2d(in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+    if len(in_shape) >= 4:
+        scratch.layer4_rn = ops.Conv2d(in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+
+    return scratch
+
+
+class ResidualConvUnit(nn.Module):
+    """Residual convolution module.
+    """
+
+    def __init__(self, features, activation, bn):
+        """Init.
+
+        Args:
+            features (int): number of features
+        """
+        super().__init__()
+
+        self.bn = bn
+
+        self.groups=1
+
+        self.conv1 = ops.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups)
+        
+        self.conv2 = ops.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups)
+
+        if self.bn == True:
+            self.bn1 = nn.BatchNorm2d(features)
+            self.bn2 = nn.BatchNorm2d(features)
+
+        self.activation = activation
+
+        self.skip_add = nn.quantized.FloatFunctional()
+
+    def forward(self, x):
+        """Forward pass.
+
+        Args:
+            x (tensor): input
+
+        Returns:
+            tensor: output
+        """
+        
+        out = self.activation(x)
+        out = self.conv1(out)
+        if self.bn == True:
+            out = self.bn1(out)
+       
+        out = self.activation(out)
+        out = self.conv2(out)
+        if self.bn == True:
+            out = self.bn2(out)
+
+        if self.groups > 1:
+            out = self.conv_merge(out)
+
+        return self.skip_add.add(out, x)
+
+
+class FeatureFusionBlock(nn.Module):
+    """Feature fusion block.
+    """
+
+    def __init__(
+        self, 
+        features, 
+        activation, 
+        deconv=False, 
+        bn=False, 
+        expand=False, 
+        align_corners=True,
+        size=None
+    ):
+        """Init.
+        
+        Args:
+            features (int): number of features
+        """
+        super(FeatureFusionBlock, self).__init__()
+
+        self.deconv = deconv
+        self.align_corners = align_corners
+
+        self.groups=1
+
+        self.expand = expand
+        out_features = features
+        if self.expand == True:
+            out_features = features // 2
+        
+        self.out_conv = ops.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1)
+
+        self.resConfUnit1 = ResidualConvUnit(features, activation, bn)
+        self.resConfUnit2 = ResidualConvUnit(features, activation, bn)
+        
+        self.skip_add = nn.quantized.FloatFunctional()
+
+        self.size=size
+
+    def forward(self, *xs, size=None):
+        """Forward pass.
+
+        Returns:
+            tensor: output
+        """
+        output = xs[0]
+
+        if len(xs) == 2:
+            res = self.resConfUnit1(xs[1])
+            output = self.skip_add.add(output, res)
+
+        output = self.resConfUnit2(output)
+
+        if (size is None) and (self.size is None):
+            modifier = {"scale_factor": 2}
+        elif size is None:
+            modifier = {"size": self.size}
+        else:
+            modifier = {"size": size}
+
+        output = nn.functional.interpolate(output, **modifier, mode="bilinear", align_corners=self.align_corners)
+        
+        output = self.out_conv(output)
+
+        return output

custom_nodes/ComfyUI-DepthAnythingV2/nodes.py

@@ -0,0 +1,189 @@
+
+import torch
+import torch.nn.functional as F
+from torchvision import transforms
+import os
+from contextlib import nullcontext
+
+import comfy.model_management as mm
+from comfy.utils import ProgressBar, load_torch_file
+import folder_paths
+
+from .depth_anything_v2.dpt import DepthAnythingV2
+
+from contextlib import nullcontext
+try:
+    from accelerate import init_empty_weights
+    from accelerate.utils import set_module_tensor_to_device
+    is_accelerate_available = True
+except:
+    is_accelerate_available = False
+    pass
+
+class DownloadAndLoadDepthAnythingV2Model:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+            "model": (
+                    [ 
+                        'depth_anything_v2_vits_fp16.safetensors',
+                        'depth_anything_v2_vits_fp32.safetensors',
+                        'depth_anything_v2_vitb_fp16.safetensors',
+                        'depth_anything_v2_vitb_fp32.safetensors',
+                        'depth_anything_v2_vitl_fp16.safetensors',
+                        'depth_anything_v2_vitl_fp32.safetensors',
+                        'depth_anything_v2_metric_hypersim_vitl_fp32.safetensors',
+                        'depth_anything_v2_metric_vkitti_vitl_fp32.safetensors'
+                    ],
+                    {
+                    "default": 'depth_anything_v2_vitl_fp32.safetensors'
+                    }),
+            },
+        }
+
+    RETURN_TYPES = ("DAMODEL",)
+    RETURN_NAMES = ("da_v2_model",)
+    FUNCTION = "loadmodel"
+    CATEGORY = "DepthAnythingV2"
+    DESCRIPTION = """
+Models autodownload to `ComfyUI\models\depthanything` from   
+https://huggingface.co/Kijai/DepthAnythingV2-safetensors/tree/main   
+   
+fp16 reduces quality by a LOT, not recommended.
+"""
+
+    def loadmodel(self, model):
+        device = mm.get_torch_device()
+        dtype = torch.float16 if "fp16" in model else torch.float32
+        model_configs = {
+            'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]},
+            'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
+            'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
+            #'vitg': {'encoder': 'vitg', 'features': 384, 'out_channels': [1536, 1536, 1536, 1536]}
+        }
+        custom_config = {
+            'model_name': model,
+        }
+        if not hasattr(self, 'model') or self.model == None or custom_config != self.current_config:
+            self.current_config = custom_config
+            download_path = os.path.join(folder_paths.models_dir, "depthanything")
+            model_path = os.path.join(download_path, model)
+
+            if not os.path.exists(model_path):
+                print(f"Downloading model to: {model_path}")
+                from huggingface_hub import snapshot_download
+                snapshot_download(repo_id="Kijai/DepthAnythingV2-safetensors", 
+                                  allow_patterns=[f"*{model}*"],
+                                  local_dir=download_path, 
+                                  local_dir_use_symlinks=False)
+
+            print(f"Loading model from: {model_path}")
+
+            if "vitl" in model:
+                encoder = "vitl"
+            elif "vitb" in model:
+                encoder = "vitb"
+            elif "vits" in model:
+                encoder = "vits"
+
+            if "hypersim" in model:
+                max_depth = 20.0
+            else:
+                max_depth = 80.0
+
+            with (init_empty_weights() if is_accelerate_available else nullcontext()):
+                if 'metric' in model:
+                    self.model = DepthAnythingV2(**{**model_configs[encoder], 'is_metric': True, 'max_depth': max_depth})
+                else:
+                    self.model = DepthAnythingV2(**model_configs[encoder])
+            
+            state_dict = load_torch_file(model_path)
+            if is_accelerate_available:
+                for key in state_dict:
+                    set_module_tensor_to_device(self.model, key, device=device, dtype=dtype, value=state_dict[key])
+            else:
+                self.model.load_state_dict(state_dict)
+
+            self.model.eval()
+ 
+        da_model = {
+            "model": self.model,
+            "dtype": dtype,
+            "is_metric": self.model.is_metric
+        }
+           
+        return (da_model,)
+    
+class DepthAnything_V2:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required": {
+            "da_model": ("DAMODEL", ),
+            "images": ("IMAGE", ),
+            },
+        }
+    
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES =("image",)
+    FUNCTION = "process"
+    CATEGORY = "DepthAnythingV2"
+    DESCRIPTION = """
+https://depth-anything-v2.github.io
+"""
+
+    def process(self, da_model, images):
+        device = mm.get_torch_device()
+        offload_device = mm.unet_offload_device()
+        model = da_model['model']
+        dtype=da_model['dtype']
+        
+        B, H, W, C = images.shape
+
+        #images = images.to(device)
+        images = images.permute(0, 3, 1, 2)
+
+        orig_H, orig_W = H, W
+        if W % 14 != 0:
+            W = W - (W % 14)
+        if H % 14 != 0:
+            H = H - (H % 14)
+        if orig_H % 14 != 0 or orig_W % 14 != 0:
+            images = F.interpolate(images, size=(H, W), mode="bilinear")
+        
+        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        normalized_images = normalize(images)
+        pbar = ProgressBar(B)
+        out = []
+        model.to(device)
+        autocast_condition = (dtype != torch.float32) and not mm.is_device_mps(device)
+        with torch.autocast(mm.get_autocast_device(device), dtype=dtype) if autocast_condition else nullcontext():
+            for img in normalized_images:
+                depth = model(img.unsqueeze(0).to(device))
+                depth = (depth - depth.min()) / (depth.max() - depth.min())
+                out.append(depth.cpu())
+                pbar.update(1)
+            model.to(offload_device)
+            depth_out = torch.cat(out, dim=0)
+            depth_out = depth_out.unsqueeze(-1).repeat(1, 1, 1, 3).cpu().float()
+        
+        final_H = (orig_H // 2) * 2
+        final_W = (orig_W // 2) * 2
+
+        
+
+        if depth_out.shape[1] != final_H or depth_out.shape[2] != final_W:
+            depth_out = F.interpolate(depth_out.permute(0, 3, 1, 2), size=(final_H, final_W), mode="bilinear").permute(0, 2, 3, 1)
+        depth_out = (depth_out - depth_out.min()) / (depth_out.max() - depth_out.min())
+        depth_out = torch.clamp(depth_out, 0, 1)
+        if da_model['is_metric']:
+            depth_out = 1 - depth_out
+        return (depth_out,)
+    
+NODE_CLASS_MAPPINGS = {
+    "DepthAnything_V2": DepthAnything_V2,
+    "DownloadAndLoadDepthAnythingV2Model": DownloadAndLoadDepthAnythingV2Model
+}
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "DepthAnything_V2": "Depth Anything V2",
+    "DownloadAndLoadDepthAnythingV2Model": "DownloadAndLoadDepthAnythingV2Model"
+}

custom_nodes/ComfyUI-DepthAnythingV2/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "comfyui-depthanythingv2"
+description = "ComfyUI nodes to use [a/DepthAnythingV2](https://depth-anything-v2.github.io/)\nNOTE:Models autodownload to ComfyUI/models/depthanything from [a/https://huggingface.co/Kijai/DepthAnythingV2-safetensors/tree/main](https://huggingface.co/Kijai/DepthAnythingV2-safetensors/tree/main)"
+version = "1.0.1"
+license = "LICENSE"
+dependencies = ["huggingface_hub", "accelerate"]
+
+[project.urls]
+Repository = "https://github.com/kijai/ComfyUI-DepthAnythingV2"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = "kijai"
+DisplayName = "ComfyUI-DepthAnythingV2"
+Icon = ""

custom_nodes/ComfyUI-DepthAnythingV2/requirements.txt

@@ -0,0 +1,2 @@
+huggingface_hub
+accelerate

custom_nodes/ComfyUI-Frame-Interpolation/ckpts/flavr/FLAVR_2x.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45509c279355866b46021b112301c4d1862e13569b4e0d58d826171d94695412
+size 504778983

custom_nodes/ComfyUI-Frame-Interpolation/ckpts/rife/rife49.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e55fd00f3cc184e3c65961f4bb827a9da022e78eed36b055242c0ac30000d533
+size 21345274

custom_nodes/ComfyUI-Frame-Interpolation/ckpts/rife/sudo_rife4_269.662_testV1_scale1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c9dff599a1c05c38cdc52930a739e7144ce8210fdd098a364e0155c1d23c27c
+size 33719173