Update README.md

README.md

@@ -43,6 +43,85 @@ Folder Structure
 # Blockers
 1) NotImplementedError: Cannot copy out of meta tensor; no data! Please use torch.nn.Module.to_empty() instead of torch.nn.Module.to() when moving module from meta to a different device.
 
+potential fix: app.diffusion.pipeline.config.py
+```python
+    @staticmethod
+    def _default_build(
+        name: str,
+        path: str,
+        dtype: str | torch.dtype,
+        device: str | torch.device,
+        shift_activations: bool
+    ) -> DiffusionPipeline:
+        if not path:
+            if name == "sdxl":
+                path = "stabilityai/stable-diffusion-xl-base-1.0"
+            elif name == "sdxl-turbo":
+                path = "stabilityai/sdxl-turbo"
+            elif name == "pixart-sigma":
+                path = "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS"
+            elif name == "flux.1-dev":
+                path = "black-forest-labs/FLUX.1-dev"
+            elif name == "flux.1-canny-dev":
+                path = "black-forest-labs/FLUX.1-Canny-dev"
+            elif name == "flux.1-depth-dev":
+                path = "black-forest-labs/FLUX.1-Depth-dev"
+            elif name == "flux.1-fill-dev":
+                path = "black-forest-labs/FLUX.1-Fill-dev"
+            elif name == "flux.1-schnell":
+                path = "black-forest-labs/FLUX.1-schnell"
+            else:
+                raise ValueError(f"Path for {name} is not specified.")
+    
+        # Instantiate the pipeline
+        if name in ["flux.1-canny-dev", "flux.1-depth-dev"]:
+            pipeline = FluxControlPipeline.from_pretrained(path, torch_dtype=dtype)
+        elif name == "flux.1-fill-dev":
+            pipeline = FluxFillPipeline.from_pretrained(path, torch_dtype=dtype)
+        elif name.startswith("sana-"):
+            if dtype == torch.bfloat16:
+                pipeline = SanaPipeline.from_pretrained(
+                    path, variant="bf16", torch_dtype=dtype, use_safetensors=True
+                )
+                pipeline.vae.to(dtype)
+                pipeline.text_encoder.to(dtype)
+            else:
+                pipeline = SanaPipeline.from_pretrained(path, torch_dtype=dtype)
+        else:
+            pipeline = AutoPipelineForText2Image.from_pretrained(path, torch_dtype=dtype)
+    
+        # Debug output
+        print(">>> DEVICE:", device)
+        print(">>> PIPELINE TYPE:", type(pipeline))
+    
+        # Try to move each component using .to_empty()
+        for name in ["unet", "transformer", "vae", "text_encoder"]:
+            module = getattr(pipeline, name, None)
+            if isinstance(module, torch.nn.Module):
+                try:
+                    print(f">>> Moving {name} to {device} using to_empty()")
+                    module.to_empty(device)
+                except Exception as e:
+                    print(f">>> WARNING: {name}.to_empty({device}) failed: {e}")
+                    try:
+                        print(f">>> Falling back to {name}.to({device})")
+                        module.to(device)
+                    except Exception as ee:
+                        print(f">>> ERROR: {name}.to({device}) also failed: {ee}")
+    
+        # Identify main model (for patching)
+        model = getattr(pipeline, "unet", None) or getattr(pipeline, "transformer", None)
+        if model is not None:
+            replace_fused_linear_with_concat_linear(model)
+            replace_up_block_conv_with_concat_conv(model)
+            if shift_activations:
+                shift_input_activations(model)
+        else:
+            print(">>> WARNING: No model (unet/transformer) found for patching")
+    
+        return pipeline
+```
+
 2) KeyError: <class 'diffusers.models.transformers.transformer_flux.FluxAttention'>