Update app.py

app.py

@@ -14,6 +14,9 @@ import paramiko
 import urllib
 import time
 import os
+from ip_adapter import IPAdapterXL
+from image_gen_aux import UpscaleWithModel
+from huggingface_hub import snapshot_download
 
 FTP_HOST = "1ink.us"
 FTP_USER = "ford442"
@@ -48,6 +51,31 @@ def upload_to_ftp(filename):
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 torch_dtype = torch.bfloat16
 
+
+repo_id = "ford442/SDXL-IP_ADAPTER"
+subfolder = "image_encoder"
+subfolder2 = "ip_adapter"
+
+# Download the entire repository
+local_repo_path = snapshot_download(repo_id=repo_id, repo_type="model")
+
+# Construct the paths to the subfolders
+local_folder = os.path.join(local_repo_path, subfolder)
+local_folder2 = os.path.join(local_repo_path, subfolder2) # Path to the ip_adapter dir
+
+print(f"Image encoder downloaded to: {local_folder}")
+print(f"IP Adapter files downloaded to: {local_folder2}")
+
+# Construct the path to the ip-adapter_sdxl.bin file
+#ip_ckpt = os.path.join(local_folder2, "ip-adapter_sdxl.bin") # Correct path
+ip_ckpt = os.path.join(local_folder2, "ip-adapter_sdxl_vit-h.bin") # Correct path
+
+print(f"IP Adapter checkpoint path: {ip_ckpt}")
+ip_model = IPAdapterXL(pipe, local_folder, ip_ckpt, device)
+
+upscaler_2 = UpscaleWithModel.from_pretrained("Kim2091/ClearRealityV1").to(torch.device("cuda:0"))
+
+
 checkpoint = "microsoft/Phi-3.5-mini-instruct"
 #vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
 vae = AutoencoderKL.from_pretrained("ford442/sdxl-vae-bf16")
@@ -186,23 +214,18 @@ def infer(
       #      device=pipe.device,
       #      generator=generator,
       #  )
-        sd_image_a = torch.load(latent_file.name)  # Load the latent
-      #  initial_latents += sd_image_a
-        #sd_image_b = pipe.vae.encode(sd_image_a.to(torch.bfloat16)).latent_dist.sample().mul_(0.18215)
-        print("-- using latent file --")
+        sd_image_a = Image.open(latent_file.name)
+        print("-- using image file --")
         print('-- generating image --')
         #with torch.no_grad():
-        sd_image = pipe(
-            prompt=enhanced_prompt,  # This conversion is fine
-            negative_prompt=negative_prompt,
-            guidance_scale=guidance_scale,
+        sd_image = ip_model.generate(
+            prompt=enhanced_prompt,
+            pil_image=sd_image_a,
+            num_samples=1,
             num_inference_steps=num_inference_steps,
-            width=width,
-            height=height,
-            latents=sd_image_a,
-          #  output='latent',
-            generator=generator
-        ).images[0]
+            guidance_scale=guidance_scale,
+            seed=seed
+        )
         rv_path = f"sd35_{seed}.png"
         sd_image[0].save(rv_path,optimize=False,compress_level=0)
         upload_to_ftp(rv_path)
@@ -329,7 +352,7 @@ with gr.Blocks(theme=gr.themes.Origin(),css=css) as demo:
             run_button = gr.Button("Run", scale=0, variant="primary")
         result = gr.Image(label="Result", show_label=False)
         with gr.Accordion("Advanced Settings", open=False):
-            latent_file = gr.File(label="Latents File (optional)")  # Add latents file input
+            latent_file = gr.File(label="Image File (optional)")  # Add latents file input
             negative_prompt = gr.Text(
                 label="Negative prompt",
                 max_lines=1,