Paste

src/pipeline.py

@@ -9,16 +9,17 @@ from huggingface_hub.constants import HF_HUB_CACHE
 from pipelines.models import TextToImageRequest
 from torch import Generator
 from torchao.quantization import quantize_, int8_weight_only
-from transformers import T5EncoderModel, CLIPTextModel
+from transformers import T5EncoderModel, CLIPTextModel, logging 
 
+
+Pipeline: TypeAlias = FluxPipeline
+torch.backends.cudnn.benchmark = True
+torch.backends.cudnn.benchmark = True
 torch._inductor.config.conv_1x1_as_mm = True
 torch._inductor.config.coordinate_descent_tuning = True
 torch._inductor.config.epilogue_fusion = False
 torch._inductor.config.coordinate_descent_check_all_directions = True
-
-Pipeline: TypeAlias = FluxPipeline
 os.environ['PYTORCH_CUDA_ALLOC_CONF']="expandable_segments:True"
-torch.backends.cudnn.benchmark = True
 
 CHECKPOINT = "jokerbit/flux.1-schnell-Robert-int8wo"
 REVISION = "5ef0012f11a863e5111ec56540302a023bc8587b"
@@ -34,22 +35,29 @@ def load_pipeline() -> Pipeline:
             use_safetensors=False,
             local_files_only=True,
             torch_dtype=torch.bfloat16)
-    
+    vae = AutoencoderTiny.from_pretrained(
+            TinyVAE, 
+            revision=TinyVAE_REV, 
+            local_files_only=True,
+            torch_dtype=torch.bfloat16)
     pipeline = FluxPipeline.from_pretrained(
         CHECKPOINT,
         revision=REVISION,
         transformer=transformer,
+        # vae=vae,
         local_files_only=True,
         torch_dtype=torch.bfloat16,
     )
+
+    pipeline.transformer.to(memory_format=torch.channels_last)
+    #    quantize_(pipeline.vae, int8_weight_only())
+    pipeline.vae.to(memory_format=torch.channels_last)
+    pipeline.vae = torch.compile(pipeline.vae, mode="max-autotune", fullgraph=True)
     pipeline.to("cuda")
-    # pipeline.transformer.to(memory_format=torch.channels_last)
-    pipeline.transformer = torch.compile(pipeline.transformer)
-    # pipeline.text_encoder.fuse_qkv_projections()
-    # pipeline.vae = torch.compile(pipeline.vae)
-    for _ in range(4):
+    
+    for _ in range(2):
         pipeline("cat", num_inference_steps=4)
-    torch.cuda.empty_cache()
+
     return pipeline
 
 @torch.inference_mode()
@@ -82,3 +90,4 @@ if __name__ == "__main__":
         infer(request, pipe_)
         stop_time = perf_counter()
         print(f"Request in {stop_time - start_time}s")
+