ABDALLALSWAITI

/

FLUX.1-dev-ControlNet-Union-Pro-2.0-fp8

like 58

Your basic code doesn't work . Let me know if you need more details.

Using the same code as in model card.

RuntimeError Traceback (most recent call last)
Cell In[3], line 7
3 width, height = control_image.size
5 prompt = "A young girl stands gracefully at the edge of a serene beach, her long, flowing hair gently tousled by the sea breeze. She wears a soft, pastel-colored dress that complements the tranquil blues and greens of the coastal scenery. The golden hues of the setting sun cast a warm glow on her face, highlighting her serene expression. The background features a vast, azure ocean with gentle waves lapping at the shore, surrounded by distant cliffs and a clear, cloudless sky. The composition emphasizes the girl's serene presence amidst the natural beauty, with a balanced blend of warm and cool tones."
----> 7 image = pipe(
8 prompt,
9 control_image=control_image,
10 width=width,
11 height=height,
12 controlnet_conditioning_scale=0.7,
13 control_guidance_end=0.8,
14 num_inference_steps=30,
15 guidance_scale=3.5,
16 generator=torch.Generator(device="cuda").manual_seed(42),
17 ).images[0]

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/torch/utils/_contextlib.py:124, in context_decorator..decorate_context(*args, **kwargs)
120 @functools.wraps(func)
121 def decorate_context(*args, **kwargs):
122 # pyrefly: ignore [bad-context-manager]
123 with ctx_factory():
--> 124 return func(*args, **kwargs)

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/diffusers/pipelines/flux/pipeline_flux_controlnet.py:1087, in FluxControlNetPipeline.call(self, prompt, prompt_2, negative_prompt, negative_prompt_2, true_cfg_scale, height, width, num_inference_steps, sigmas, guidance_scale, control_guidance_start, control_guidance_end, control_image, control_mode, controlnet_conditioning_scale, num_images_per_prompt, generator, latents, prompt_embeds, pooled_prompt_embeds, ip_adapter_image, ip_adapter_image_embeds, negative_ip_adapter_image, negative_ip_adapter_image_embeds, negative_prompt_embeds, negative_pooled_prompt_embeds, output_type, return_dict, joint_attention_kwargs, callback_on_step_end, callback_on_step_end_tensor_inputs, max_sequence_length)
1084 cond_scale = controlnet_cond_scale * controlnet_keep[i]
1086 # controlnet
-> 1087 controlnet_block_samples, controlnet_single_block_samples = self.controlnet(
1088 hidden_states=latents,
1089 controlnet_cond=control_image,
1090 controlnet_mode=control_mode,
1091 conditioning_scale=cond_scale,
1092 timestep=timestep / 1000,
1093 guidance=guidance,
1094 pooled_projections=pooled_prompt_embeds,
1095 encoder_hidden_states=prompt_embeds,
1096 txt_ids=text_ids,
1097 img_ids=latent_image_ids,
1098 joint_attention_kwargs=self.joint_attention_kwargs,
1099 return_dict=False,
1100 )
1102 guidance = (
1103 torch.tensor([guidance_scale], device=device) if self.transformer.config.guidance_embeds else None
1104 )
1105 guidance = guidance.expand(latents.shape[0]) if guidance is not None else None

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/torch/nn/modules/module.py:1779, in Module._wrapped_call_impl(self, *args, **kwargs)
1777 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1778 else:
-> 1779 return self._call_impl(*args, **kwargs)

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/torch/nn/modules/module.py:1790, in Module._call_impl(self, *args, **kwargs)
1785 # If we don't have any hooks, we want to skip the rest of the logic in
1786 # this function, and just call forward.
1787 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1788 or _global_backward_pre_hooks or _global_backward_hooks
1789 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1790 return forward_call(*args, **kwargs)
1792 result = None
1793 called_always_called_hooks = set()

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/accelerate/hooks.py:192, in add_hook_to_module..new_forward(module, *args, **kwargs)
190 output = module._old_forward(*args, **kwargs)
191 else:
--> 192 output = module._old_forward(*args, **kwargs)
193 return module._hf_hook.post_forward(module, output)

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/diffusers/utils/peft_utils.py:315, in apply_lora_scale..decorator..wrapper(self, *args, **kwargs)
311 scale_lora_layers(self, lora_scale)
313 try:
314 # Execute the forward pass
--> 315 result = forward_fn(self, *args, **kwargs)
316 return result
317 finally:
318 # Always unscale, even if forward pass raises an exception

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/diffusers/models/controlnets/controlnet_flux.py:205, in FluxControlNetModel.forward(self, hidden_states, controlnet_cond, controlnet_mode, conditioning_scale, encoder_hidden_states, pooled_projections, timestep, img_ids, txt_ids, guidance, joint_attention_kwargs, return_dict)
157 @apply_lora_scale("joint_attention_kwargs")
158 def forward(
159 self,
(...) 171 return_dict: bool = True,
172 ) -> torch.FloatTensor | Transformer2DModelOutput:
173 """
174 The [FluxTransformer2DModel] forward method.
175
(...) 203 tuple where the first element is the sample tensor.
204 """
--> 205 hidden_states = self.x_embedder(hidden_states)
207 if self.input_hint_block is not None:
208 controlnet_cond = self.input_hint_block(controlnet_cond)

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/torch/nn/modules/module.py:1779, in Module._wrapped_call_impl(self, *args, **kwargs)
1777 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1778 else:
-> 1779 return self._call_impl(*args, **kwargs)

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/torch/nn/modules/module.py:1790, in Module._call_impl(self, *args, **kwargs)
1785 # If we don't have any hooks, we want to skip the rest of the logic in
1786 # this function, and just call forward.
1787 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1788 or _global_backward_pre_hooks or _global_backward_hooks
1789 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1790 return forward_call(*args, **kwargs)
1792 result = None
1793 called_always_called_hooks = set()

File ~/anaconda3/envs/huggingFace/lib/python3.12/site-packages/torch/nn/modules/linear.py:134, in Linear.forward(self, input)
130 def forward(self, input: Tensor) -> Tensor:
131 """
132 Runs the forward pass.
133 """
--> 134 return F.linear(input, self.weight, self.bias)

RuntimeError: mat1 and mat2 must have the same dtype, but got BFloat16 and Float8_e4m3fn