Try disabling VAE slicing to see if that removes the CUDA incompatible half type.

utils/pipes.py

@@ -9,6 +9,7 @@ def create_stable_diffusion_xl_pipeline(device):
         "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
     )
     pipe = pipe.to(device)
+    pipe.vae.disable_vae_slicing()  # Experimental
     pipe.vae.disable_tiling()
     tiling.enable_circular_tiling([pipe.vae, pipe.text_encoder, pipe.unet])
     return pipe
@@ -19,6 +20,7 @@ def create_stable_diffusion_xl_img2img_pipe(device):
         "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
     )
     pipe = pipe.to(device)
+    pipe.vae.disable_vae_slicing()  # Experimental
     pipe.vae.disable_tiling()
     tiling.enable_circular_tiling([pipe.vae, pipe.text_encoder_2, pipe.unet])
     return pipe