update

hf_demo.py

@@ -16,70 +16,93 @@ pipe = DiffusionPipeline.from_pretrained("rhfeiyang/art-free-diffusion-v1",
 from inference import get_lora_network, inference, get_validation_dataloader
 lora_map = {
     "None": "None",
-    "Andre Derain": "andre-derain_subset1",
-    "Vincent van Gogh": "van_gogh_subset1",
-    "Andy Warhol": "andy_subset1",
+    "Andre Derain (fauvism)": "andre-derain_subset1",
+    "Vincent van Gogh (post impressionism)": "van_gogh_subset1",
+    "Andy Warhol (pop art)": "andy_subset1",
     "Walter Battiss": "walter-battiss_subset2",
-    "Camille Corot": "camille-corot_subset1",
-    "Claude Monet": "monet_subset2",
-    "Pablo Picasso": "picasso_subset1",
+    "Camille Corot (realism)": "camille-corot_subset1",
+    "Claude Monet (impressionism)": "monet_subset2",
+    "Pablo Picasso (cubism)": "picasso_subset1",
     "Jackson Pollock": "jackson-pollock_subset1",
-    "Gerhard Richter": "gerhard-richter_subset1",
+    "Gerhard Richter (abstract expressionism)": "gerhard-richter_subset1",
     "M.C. Escher": "m.c.-escher_subset1",
     "Albert Gleizes": "albert-gleizes_subset1",
-    "Hokusai": "katsushika-hokusai_subset1",
+    "Hokusai (ukiyo-e)": "katsushika-hokusai_subset1",
     "Wassily Kandinsky": "kandinsky_subset1",
-    "Gustav Klimt": "klimt_subset3",
+    "Gustav Klimt (art nouveau)": "klimt_subset3",
     "Roy Lichtenstein": "roy-lichtenstein_subset1",
-    "Henri Matisse": "henri-matisse_subset1",
+    "Henri Matisse (abstract expressionism)": "henri-matisse_subset1",
     "Joan Miro": "joan-miro_subset2",
 }
+
 @spaces.GPU
-def demo_inference_gen(adapter_choice:str, prompt:str, samples:int=1,seed:int=0, steps=50, guidance_scale=7.5):
+def demo_inference_gen_artistic(adapter_choice:str, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, adapter_scale=1.0):
     adapter_path = lora_map[adapter_choice]
     if adapter_path not in [None, "None"]:
         adapter_path = f"data/Art_adapters/{adapter_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt"
         style_prompt="sks art"
     else:
         style_prompt=None
-    prompts = [prompt]*samples
+    prompts = [prompt]
     infer_loader = get_validation_dataloader(prompts,num_workers=0)
     network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype)["network"]
 
     pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,
-                            height=512, width=512, scales=[1.0],
+                            height=512, width=512, scales=[adapter_scale],
                             save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,
                             start_noise=-1, show=False, style_prompt=style_prompt, no_load=True,
-                            from_scratch=True, device=device, weight_dtype=dtype)[0][1.0]
+                            from_scratch=True, device=device, weight_dtype=dtype)[0][1.0][0]
     return pred_images
+
 @spaces.GPU
-def demo_inference_stylization(adapter_path:str, prompts:list, image:list, start_noise=800,seed:int=0):
-    infer_loader = get_validation_dataloader(prompts, image,num_workers=0)
-    network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype)["network"]
+def demo_inference_gen_ori( prompt:str, seed:int=0, steps=50, guidance_scale=7.5):
+    style_prompt=None
+    prompts = [prompt]
+    infer_loader = get_validation_dataloader(prompts,num_workers=0)
+    network = get_lora_network(pipe.unet, "None", weight_dtype=dtype)["network"]
+
     pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,
-                            height=512, width=512, scales=[0.,1.],
-                            save_dir=None, seed=seed,steps=20, guidance_scale=7.5,
-                            start_noise=start_noise, show=True, style_prompt="sks art", no_load=True,
-                            from_scratch=False, device=device, weight_dtype=dtype)[0][1.0]
+                            height=512, width=512, scales=[0.0],
+                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,
+                            start_noise=-1, show=False, style_prompt=style_prompt, no_load=True,
+                            from_scratch=True, device=device, weight_dtype=dtype)[0][0.0][0]
     return pred_images
 
-# def infer(prompt, samples, steps, scale, seed):
-#     generator = torch.Generator(device=device).manual_seed(seed)
-#     images_list = pipe(  # type: ignore
-#         [prompt] * samples,
-#         num_inference_steps=steps,
-#         guidance_scale=scale,
-#         generator=generator,
-#     )
-#     images = []
-#     safe_image = Image.open(r"data/unsafe.png")
-#     print(images_list)
-#     for i, image in enumerate(images_list["images"]):  # type: ignore
-#         if images_list["nsfw_content_detected"][i]:  # type: ignore
-#             images.append(safe_image)
-#         else:
-#             images.append(image)
-#     return images
+
+@spaces.GPU
+def demo_inference_stylization_ori(ref_image, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, start_noise=800):
+    style_prompt=None
+    prompts = [prompt]
+    # convert np to pil
+    ref_image = [Image.fromarray(ref_image)]
+    network = get_lora_network(pipe.unet, "None", weight_dtype=dtype)["network"]
+    infer_loader = get_validation_dataloader(prompts, ref_image,num_workers=0)
+    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,
+                            height=512, width=512, scales=[0.0],
+                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,
+                            start_noise=start_noise, show=False, style_prompt=style_prompt, no_load=True,
+                            from_scratch=False, device=device, weight_dtype=dtype)[0][0.0][0]
+    return pred_images
+
+@spaces.GPU
+def demo_inference_stylization_artistic(ref_image, adapter_choice:str, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, adapter_scale=1.0,start_noise=800):
+    adapter_path = lora_map[adapter_choice]
+    if adapter_path not in [None, "None"]:
+        adapter_path = f"data/Art_adapters/{adapter_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt"
+        style_prompt="sks art"
+    else:
+        style_prompt=None
+    prompts = [prompt]
+    # convert np to pil
+    ref_image = [Image.fromarray(ref_image)]
+    network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype)["network"]
+    infer_loader = get_validation_dataloader(prompts, ref_image,num_workers=0)
+    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,
+                            height=512, width=512, scales=[adapter_scale],
+                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,
+                            start_noise=start_noise, show=False, style_prompt=style_prompt, no_load=True,
+                            from_scratch=False, device=device, weight_dtype=dtype)[0][1.0][0]
+    return pred_images
 
 
 
@@ -92,62 +115,119 @@ with block:
         gr.Markdown("(More features in development...)")
         with gr.Row():
             text = gr.Textbox(
-                label="Enter your prompt",
+                label="Enter your prompt(long and detailed would be better):",
                 max_lines=2,
-                placeholder="Enter your prompt",
-                container=False,
+                placeholder="Enter your prompt(long and detailed would be better)",
+                container=True,
                 value="Park with cherry blossom trees, picnicker’s and a clear blue pond.",
             )
 
+        with gr.Tab('Generation'):
+            with gr.Row():
+                with gr.Column():
+                    # gr.Markdown("## Art-Free Generation")
+                    # gr.Markdown("Generate images from text prompts.")
+
+                    gallery_gen_ori = gr.Image(
+                        label="W/O Adapter",
+                        show_label=True,
+                        elem_id="gallery",
+                        height="auto"
+                    )
+
+
+                with gr.Column():
+                    # gr.Markdown("## Art-Free Generation")
+                    # gr.Markdown("Generate images from text prompts.")
+                    gallery_gen_art = gr.Image(
+                        label="W/ Adapter",
+                        show_label=True,
+                        elem_id="gallery",
+                        height="auto"
+                    )
+
+
+            with gr.Row():
+                btn_gen_ori = gr.Button("Art-Free Generate", scale=1)
+                btn_gen_art = gr.Button("Artistic Generate", scale=1)
+
+
+        with gr.Tab('Stylization'):
+            with gr.Row():
+
+                with gr.Column():
+                    # gr.Markdown("## Art-Free Generation")
+                    # gr.Markdown("Generate images from text prompts.")
+
+                    gallery_stylization_ref = gr.Image(
+                        label="Ref Image",
+                        show_label=True,
+                        elem_id="gallery",
+                        height="auto",
+                        scale=1,
+                    )
+                with gr.Column(scale=2):
+                    with gr.Row():
+                        with gr.Column():
+                            # gr.Markdown("## Art-Free Generation")
+                            # gr.Markdown("Generate images from text prompts.")
+
+                            gallery_stylization_ori = gr.Image(
+                                label="W/O Adapter",
+                                show_label=True,
+                                elem_id="gallery",
+                                height="auto",
+                                scale=1,
+                            )
+
+
+                        with gr.Column():
+                            # gr.Markdown("## Art-Free Generation")
+                            # gr.Markdown("Generate images from text prompts.")
+                            gallery_stylization_art = gr.Image(
+                                label="W/ Adapter",
+                                show_label=True,
+                                elem_id="gallery",
+                                height="auto",
+                                scale=1,
+                            )
+                    start_timestep = gr.Slider(label="Adapter Timestep", minimum=0, maximum=1000, value=800, step=1)
+            with gr.Row():
+                btn_style_ori = gr.Button("Art-Free Stylization", scale=1)
+                btn_style_art = gr.Button("Artistic Stylization", scale=1)
 
 
-            btn = gr.Button("Run", scale=0)
-        gallery = gr.Gallery(
-            label="Generated images",
-            show_label=False,
-            elem_id="gallery",
-            columns=[1],
-        )
-
-        advanced_button = gr.Button("Advanced options", elem_id="advanced-btn")
-
-        with gr.Row(elem_id="advanced-options"):
+        with gr.Row():
+            # with gr.Column():
+            # samples = gr.Slider(label="Images", minimum=1, maximum=4, value=1, step=1, scale=1)
+            scale = gr.Slider(
+                label="Guidance Scale", minimum=0, maximum=20, value=7.5, step=0.1
+            )
+            # with gr.Column():
             adapter_choice = gr.Dropdown(
                 label="Select Art Adapter",
-                choices=["None", "Andre Derain","Vincent van Gogh","Andy Warhol", "Walter Battiss",
-                         "Camille Corot", "Claude Monet", "Pablo Picasso",
-                         "Jackson Pollock", "Gerhard Richter", "M.C. Escher",
-                         "Albert Gleizes", "Hokusai", "Wassily Kandinsky", "Gustav Klimt", "Roy Lichtenstein",
-                         "Henri Matisse", "Joan Miro"
-                         ],
-                value="None"
+                choices=[ "Andre Derain (fauvism)","Vincent van Gogh (post impressionism)","Andy Warhol (pop art)",
+                          "Camille Corot (realism)", "Claude Monet (impressionism)", "Pablo Picasso (cubism)", "Gerhard Richter (abstract expressionism)",
+                          "Hokusai (ukiyo-e)", "Gustav Klimt (art nouveau)", "Henri Matisse (abstract expressionism)",
+                          "Walter Battiss", "Jackson Pollock",  "M.C. Escher", "Albert Gleizes",  "Wassily Kandinsky",
+                          "Roy Lichtenstein", "Joan Miro"
+                          ],
+                value="Andre Derain (fauvism)",
+                scale=1
             )
-            # print(adapter_choice[0])
-            # lora_path = lora_map[adapter_choice.value]
-            # if lora_path is not None:
-            #     lora_path = f"data/Art_adapters/{lora_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt"
 
-            samples = gr.Slider(label="Images", minimum=1, maximum=4, value=1, step=1)
+        with gr.Row():
             steps = gr.Slider(label="Steps", minimum=1, maximum=50, value=20, step=1)
-            scale = gr.Slider(
-                label="Guidance Scale", minimum=0, maximum=50, value=7.5, step=0.1
-            )
-            print(scale)
-            seed = gr.Slider(
-                label="Seed",
-                minimum=0,
-                maximum=2147483647,
-                step=1,
-                randomize=True,
-            )
+            adapter_scale = gr.Slider(label="Stylization Scale", minimum=0, maximum=1.5, value=1., step=0.1, scale=1)
+
+        with gr.Row():
+            seed = gr.Slider(label="Seed",minimum=0,maximum=2147483647,step=1,randomize=True,scale=1)
 
-        gr.on([text.submit, btn.click], demo_inference_gen, inputs=[adapter_choice, text, samples, seed, steps, scale], outputs=gallery)
-        advanced_button.click(
-            None,
-            [],
-            text,
-        )
 
+        gr.on([btn_gen_ori.click], demo_inference_gen_ori, inputs=[text, seed, steps, scale], outputs=gallery_gen_ori)
+        gr.on([btn_gen_art.click], demo_inference_gen_artistic, inputs=[adapter_choice, text, seed, steps, scale, adapter_scale], outputs=gallery_gen_art)
 
+        gr.on([btn_style_ori.click], demo_inference_stylization_ori, inputs=[gallery_stylization_ref, text, seed, steps, scale, start_timestep], outputs=gallery_stylization_ori)
+        gr.on([btn_style_art.click], demo_inference_stylization_artistic, inputs=[gallery_stylization_ref, adapter_choice, text, seed, steps, scale, adapter_scale, start_timestep], outputs=gallery_stylization_art)
 
-block.launch()
+block.launch(sharing=True)

hf_demo_test.ipynb

@@ -45,7 +45,9 @@
    },
    "outputs": [],
    "source": [
-    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16"
    ]
   },
   {
@@ -70,7 +72,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "9df8347307674ba8afb0250e23109aa1",
+       "model_id": "acc42f294243439798e4d77d1a59296d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -83,8 +85,8 @@
     }
    ],
    "source": [
-    "pipe = DiffusionPipeline.from_pretrained(\"rhfeiyang/art-free-diffusion-v1\",).to(\"cuda\")\n",
-    "device = \"cuda\""
+    "pipe = DiffusionPipeline.from_pretrained(\"rhfeiyang/art-free-diffusion-v1\",\n",
+    "                                         torch_dtype=dtype).to(device)"
    ]
   },
   {
@@ -102,77 +104,105 @@
     "from inference import get_lora_network, inference, get_validation_dataloader\n",
     "lora_map = {\n",
     "    \"None\": \"None\",\n",
-    "    \"Andre Derain\": \"andre-derain_subset1\",\n",
-    "    \"Vincent van Gogh\": \"van_gogh_subset1\",\n",
-    "    \"Andy Warhol\": \"andy_subset1\",\n",
+    "    \"Andre Derain (fauvism)\": \"andre-derain_subset1\",\n",
+    "    \"Vincent van Gogh (post impressionism)\": \"van_gogh_subset1\",\n",
+    "    \"Andy Warhol (pop art)\": \"andy_subset1\",\n",
     "    \"Walter Battiss\": \"walter-battiss_subset2\",\n",
-    "    \"Camille Corot\": \"camille-corot_subset1\",\n",
-    "    \"Claude Monet\": \"monet_subset2\",\n",
-    "    \"Pablo Picasso\": \"picasso_subset1\",\n",
+    "    \"Camille Corot (realism)\": \"camille-corot_subset1\",\n",
+    "    \"Claude Monet (impressionism)\": \"monet_subset2\",\n",
+    "    \"Pablo Picasso (cubism)\": \"picasso_subset1\",\n",
     "    \"Jackson Pollock\": \"jackson-pollock_subset1\",\n",
-    "    \"Gerhard Richter\": \"gerhard-richter_subset1\",\n",
+    "    \"Gerhard Richter (abstract expressionism)\": \"gerhard-richter_subset1\",\n",
     "    \"M.C. Escher\": \"m.c.-escher_subset1\",\n",
     "    \"Albert Gleizes\": \"albert-gleizes_subset1\",\n",
-    "    \"Hokusai\": \"katsushika-hokusai_subset1\",\n",
+    "    \"Hokusai (ukiyo-e)\": \"katsushika-hokusai_subset1\",\n",
     "    \"Wassily Kandinsky\": \"kandinsky_subset1\",\n",
-    "    \"Gustav Klimt\": \"klimt_subset3\",\n",
+    "    \"Gustav Klimt (art nouveau)\": \"klimt_subset3\",\n",
     "    \"Roy Lichtenstein\": \"roy-lichtenstein_subset1\",\n",
-    "    \"Henri Matisse\": \"henri-matisse_subset1\",\n",
+    "    \"Henri Matisse (abstract expressionism)\": \"henri-matisse_subset1\",\n",
     "    \"Joan Miro\": \"joan-miro_subset2\",\n",
     "}\n",
     "\n",
-    "def demo_inference_gen(adapter_choice:str, prompt:str, samples:int=1,seed:int=0, steps=50, guidance_scale=7.5):\n",
+    "\n",
+    "\n",
+    "def demo_inference_gen_artistic(adapter_choice:str, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, adapter_scale=1.0):\n",
     "    adapter_path = lora_map[adapter_choice]\n",
     "    if adapter_path not in [None, \"None\"]:\n",
     "        adapter_path = f\"data/Art_adapters/{adapter_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt\"\n",
+    "        style_prompt=\"sks art\"\n",
+    "    else:\n",
+    "        style_prompt=None\n",
+    "    prompts = [prompt]\n",
+    "    infer_loader = get_validation_dataloader(prompts,num_workers=0)\n",
+    "    network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype)[\"network\"]\n",
     "\n",
-    "    prompts = [prompt]*samples\n",
-    "    infer_loader = get_validation_dataloader(prompts)\n",
-    "    network = get_lora_network(pipe.unet, adapter_path)[\"network\"]\n",
     "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
-    "                            height=512, width=512, scales=[1.0],\n",
+    "                            height=512, width=512, scales=[adapter_scale],\n",
     "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
-    "                            start_noise=-1, show=False, style_prompt=\"sks art\", no_load=True,\n",
-    "                            from_scratch=True)[0][1.0]\n",
+    "                            start_noise=-1, show=False, style_prompt=style_prompt, no_load=True,\n",
+    "                            from_scratch=True, device=device, weight_dtype=dtype)[0][1.0][0]\n",
     "    return pred_images\n",
     "\n",
-    "def demo_inference_stylization(adapter_path:str, prompts:list, image:list, start_noise=800,seed:int=0):\n",
-    "    infer_loader = get_validation_dataloader(prompts, image)\n",
-    "    network = get_lora_network(pipe.unet, adapter_path,\"all_up\")[\"network\"]\n",
+    "\n",
+    "def demo_inference_gen_ori( prompt:str, seed:int=0, steps=50, guidance_scale=7.5):\n",
+    "    style_prompt=None\n",
+    "    prompts = [prompt]\n",
+    "    infer_loader = get_validation_dataloader(prompts,num_workers=0)\n",
+    "    network = get_lora_network(pipe.unet, \"None\", weight_dtype=dtype)[\"network\"]\n",
+    "\n",
     "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
-    "                            height=512, width=512, scales=[0.,1.],\n",
-    "                            save_dir=None, seed=seed,steps=20, guidance_scale=7.5,\n",
-    "                            start_noise=start_noise, show=True, style_prompt=\"sks art\", no_load=True,\n",
-    "                            from_scratch=False)\n",
+    "                            height=512, width=512, scales=[0.0],\n",
+    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
+    "                            start_noise=-1, show=False, style_prompt=style_prompt, no_load=True,\n",
+    "                            from_scratch=True, device=device, weight_dtype=dtype)[0][0.0][0]\n",
     "    return pred_images\n",
     "\n",
-    "# def infer(prompt, samples, steps, scale, seed):\n",
-    "#     generator = torch.Generator(device=device).manual_seed(seed)\n",
-    "#     images_list = pipe(  # type: ignore\n",
-    "#         [prompt] * samples,\n",
-    "#         num_inference_steps=steps,\n",
-    "#         guidance_scale=scale,\n",
-    "#         generator=generator,\n",
-    "#     )\n",
-    "#     images = []\n",
-    "#     safe_image = Image.open(r\"data/unsafe.png\")\n",
-    "#     print(images_list)\n",
-    "#     for i, image in enumerate(images_list[\"images\"]):  # type: ignore\n",
-    "#         if images_list[\"nsfw_content_detected\"][i]:  # type: ignore\n",
-    "#             images.append(safe_image)\n",
-    "#         else:\n",
-    "#             images.append(image)\n",
-    "#     return images\n"
+    "\n",
+    "\n",
+    "def demo_inference_stylization_ori(ref_image, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, start_noise=800):\n",
+    "    style_prompt=None\n",
+    "    prompts = [prompt]\n",
+    "    # convert np to pil\n",
+    "    ref_image = [Image.fromarray(ref_image)]\n",
+    "    network = get_lora_network(pipe.unet, \"None\", weight_dtype=dtype)[\"network\"]\n",
+    "    infer_loader = get_validation_dataloader(prompts, ref_image,num_workers=0)\n",
+    "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
+    "                            height=512, width=512, scales=[0.0],\n",
+    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
+    "                            start_noise=start_noise, show=False, style_prompt=style_prompt, no_load=True,\n",
+    "                            from_scratch=False, device=device, weight_dtype=dtype)[0][0.0][0]\n",
+    "    return pred_images\n",
+    "\n",
+    "\n",
+    "def demo_inference_stylization_artistic(ref_image, adapter_choice:str, prompt:str, seed:int=0, steps=50, guidance_scale=7.5, adapter_scale=1.0,start_noise=800):\n",
+    "    adapter_path = lora_map[adapter_choice]\n",
+    "    if adapter_path not in [None, \"None\"]:\n",
+    "        adapter_path = f\"data/Art_adapters/{adapter_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt\"\n",
+    "        style_prompt=\"sks art\"\n",
+    "    else:\n",
+    "        style_prompt=None\n",
+    "    prompts = [prompt]\n",
+    "    # convert np to pil\n",
+    "    ref_image = [Image.fromarray(ref_image)]\n",
+    "    network = get_lora_network(pipe.unet, adapter_path, weight_dtype=dtype)[\"network\"]\n",
+    "    infer_loader = get_validation_dataloader(prompts, ref_image,num_workers=0)\n",
+    "    pred_images = inference(network, pipe.tokenizer, pipe.text_encoder, pipe.vae, pipe.unet, pipe.scheduler, infer_loader,\n",
+    "                            height=512, width=512, scales=[adapter_scale],\n",
+    "                            save_dir=None, seed=seed,steps=steps, guidance_scale=guidance_scale,\n",
+    "                            start_noise=start_noise, show=False, style_prompt=style_prompt, no_load=True,\n",
+    "                            from_scratch=False, device=device, weight_dtype=dtype)[0][1.0][0]\n",
+    "    return pred_images\n",
+    "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 15,
    "id": "aa33e9d104023847",
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2024-12-09T12:09:39.339583Z",
-     "start_time": "2024-12-09T12:09:38.953936Z"
+     "end_time": "2024-12-10T02:56:13.419303Z",
+     "start_time": "2024-12-10T02:56:13.002796Z"
     }
    },
    "outputs": [
@@ -180,9 +210,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "<gradio.components.slider.Slider object at 0x7fa12d3a5280>\n",
-      "Running on local URL:  http://127.0.0.1:7876\n",
-      "Running on public URL: https://be7cce8fec75395c82.gradio.live\n",
+      "Running on local URL:  http://127.0.0.1:7869\n",
+      "\n",
+      "Thanks for being a Gradio user! If you have questions or feedback, please join our Discord server and chat with us: https://discord.gg/feTf9x3ZSB\n",
+      "Running on public URL: https://0fd0c028b349b76a72.gradio.live\n",
       "\n",
       "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
      ]
@@ -190,7 +221,7 @@
     {
      "data": {
       "text/html": [
-       "<div><iframe src=\"https://be7cce8fec75395c82.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+       "<div><iframe src=\"https://0fd0c028b349b76a72.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
@@ -203,103 +234,135 @@
      "data": {
       "text/plain": []
      },
-     "execution_count": 6,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Train method: None\n",
-      "Rank: 1, Alpha: 1\n",
-      "create LoRA for U-Net: 0 modules.\n",
-      "save dir: None\n",
-      "['Park with cherry blossom trees, picnicker’s and a clear blue pond in the style of sks art'], seed=949192390\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/data/vision/torralba/selfmanaged/torralba/scratch/jomat/sam_dataset/miniforge3/envs/diffusion/lib/python3.9/site-packages/torch/nn/modules/conv.py:456: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at /opt/conda/conda-bld/pytorch_1712608883701/work/aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)\n",
-      "  return F.conv2d(input, weight, bias, self.stride,\n",
-      "\n",
-      "00%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:03<00:00,  6.90it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Time taken for one batch, Art Adapter scale=1.0: 3.2747044563293457\n"
-     ]
     }
    ],
    "source": [
     "block = gr.Blocks()\n",
     "# Direct infer\n",
+    "# Direct infer\n",
     "with block:\n",
     "    with gr.Group():\n",
     "        gr.Markdown(\" # Art-Free Diffusion Demo\")\n",
+    "        gr.Markdown(\"(More features in development...)\")\n",
     "        with gr.Row():\n",
     "            text = gr.Textbox(\n",
-    "                label=\"Enter your prompt\",\n",
+    "                label=\"Enter your prompt(long and detailed would be better):\",\n",
     "                max_lines=2,\n",
-    "                placeholder=\"Enter your prompt\",\n",
-    "                container=False,\n",
+    "                placeholder=\"Enter your prompt(long and detailed would be better)\",\n",
+    "                container=True,\n",
     "                value=\"Park with cherry blossom trees, picnicker’s and a clear blue pond.\",\n",
     "            )\n",
-    "            \n",
     "\n",
-    "            \n",
-    "            btn = gr.Button(\"Run\", scale=0)\n",
-    "        gallery = gr.Gallery(\n",
-    "            label=\"Generated images\",\n",
-    "            show_label=False,\n",
-    "            elem_id=\"gallery\",\n",
-    "            columns=[2],\n",
-    "        )\n",
+    "        with gr.Tab('Generation'):\n",
+    "            with gr.Row():\n",
+    "                with gr.Column():\n",
+    "                    # gr.Markdown(\"## Art-Free Generation\")\n",
+    "                    # gr.Markdown(\"Generate images from text prompts.\")\n",
     "\n",
-    "        advanced_button = gr.Button(\"Advanced options\", elem_id=\"advanced-btn\")\n",
+    "                    gallery_gen_ori = gr.Image(\n",
+    "                        label=\"W/O Adapter\",\n",
+    "                        show_label=True,\n",
+    "                        elem_id=\"gallery\",\n",
+    "                        height=\"auto\"\n",
+    "                    )\n",
     "\n",
-    "        with gr.Row(elem_id=\"advanced-options\"):\n",
-    "            adapter_choice = gr.Dropdown(\n",
-    "                label=\"Choose adapter\",\n",
-    "                choices=[\"None\", \"Andre Derain\",\"Vincent van Gogh\",\"Andy Warhol\", \"Walter Battiss\",\n",
-    "                         \"Camille Corot\", \"Claude Monet\", \"Pablo Picasso\",\n",
-    "                         \"Jackson Pollock\", \"Gerhard Richter\", \"M.C. Escher\",\n",
-    "                         \"Albert Gleizes\", \"Hokusai\", \"Wassily Kandinsky\", \"Gustav Klimt\", \"Roy Lichtenstein\",\n",
-    "                         \"Henri Matisse\", \"Joan Miro\"\n",
-    "                         ],\n",
-    "                value=\"None\"\n",
-    "            )\n",
-    "            # print(adapter_choice[0])\n",
-    "            # lora_path = lora_map[adapter_choice.value]\n",
-    "            # if lora_path is not None:\n",
-    "            #     lora_path = f\"data/Art_adapters/{lora_path}/adapter_alpha1.0_rank1_all_up_1000steps.pt\"\n",
     "\n",
-    "            samples = gr.Slider(label=\"Images\", minimum=1, maximum=4, value=1, step=1)\n",
-    "            steps = gr.Slider(label=\"Steps\", minimum=1, maximum=50, value=20, step=1)\n",
+    "                with gr.Column():\n",
+    "                    # gr.Markdown(\"## Art-Free Generation\")\n",
+    "                    # gr.Markdown(\"Generate images from text prompts.\")\n",
+    "                    gallery_gen_art = gr.Image(\n",
+    "                        label=\"W/ Adapter\",\n",
+    "                        show_label=True,\n",
+    "                        elem_id=\"gallery\",\n",
+    "                        height=\"auto\"\n",
+    "                    )\n",
+    "\n",
+    "\n",
+    "            with gr.Row():\n",
+    "                btn_gen_ori = gr.Button(\"Art-Free Generate\", scale=1)\n",
+    "                btn_gen_art = gr.Button(\"Artistic Generate\", scale=1)\n",
+    "\n",
+    "\n",
+    "        with gr.Tab('Stylization'):\n",
+    "            with gr.Row():\n",
+    "\n",
+    "                with gr.Column():\n",
+    "                    # gr.Markdown(\"## Art-Free Generation\")\n",
+    "                    # gr.Markdown(\"Generate images from text prompts.\")\n",
+    "\n",
+    "                    gallery_stylization_ref = gr.Image(\n",
+    "                        label=\"Ref Image\",\n",
+    "                        show_label=True,\n",
+    "                        elem_id=\"gallery\",\n",
+    "                        height=\"auto\",\n",
+    "                        scale=1,\n",
+    "                    )\n",
+    "                with gr.Column(scale=2):\n",
+    "                    with gr.Row():\n",
+    "                        with gr.Column():\n",
+    "                            # gr.Markdown(\"## Art-Free Generation\")\n",
+    "                            # gr.Markdown(\"Generate images from text prompts.\")\n",
+    "    \n",
+    "                            gallery_stylization_ori = gr.Image(\n",
+    "                                label=\"W/O Adapter\",\n",
+    "                                show_label=True,\n",
+    "                                elem_id=\"gallery\",\n",
+    "                                height=\"auto\",\n",
+    "                                scale=1,\n",
+    "                            )\n",
+    "    \n",
+    "    \n",
+    "                        with gr.Column():\n",
+    "                            # gr.Markdown(\"## Art-Free Generation\")\n",
+    "                            # gr.Markdown(\"Generate images from text prompts.\")\n",
+    "                            gallery_stylization_art = gr.Image(\n",
+    "                                label=\"W/ Adapter\",\n",
+    "                                show_label=True,\n",
+    "                                elem_id=\"gallery\",\n",
+    "                                height=\"auto\",\n",
+    "                                scale=1,\n",
+    "                            )\n",
+    "                    start_timestep = gr.Slider(label=\"Adapter Timestep\", minimum=0, maximum=1000, value=800, step=1)\n",
+    "            with gr.Row():\n",
+    "                btn_style_ori = gr.Button(\"Art-Free Stylization\", scale=1)\n",
+    "                btn_style_art = gr.Button(\"Artistic Stylization\", scale=1)\n",
+    "\n",
+    "\n",
+    "        with gr.Row():\n",
+    "            # with gr.Column():\n",
+    "            # samples = gr.Slider(label=\"Images\", minimum=1, maximum=4, value=1, step=1, scale=1)\n",
     "            scale = gr.Slider(\n",
-    "                label=\"Guidance Scale\", minimum=0, maximum=50, value=7.5, step=0.1\n",
+    "                label=\"Guidance Scale\", minimum=0, maximum=20, value=7.5, step=0.1\n",
     "            )\n",
-    "            print(scale)\n",
-    "            seed = gr.Slider(\n",
-    "                label=\"Seed\",\n",
-    "                minimum=0,\n",
-    "                maximum=2147483647,\n",
-    "                step=1,\n",
-    "                randomize=True,\n",
+    "            # with gr.Column():\n",
+    "            adapter_choice = gr.Dropdown(\n",
+    "                label=\"Select Art Adapter\",\n",
+    "                choices=[ \"Andre Derain (fauvism)\",\"Vincent van Gogh (post impressionism)\",\"Andy Warhol (pop art)\",\n",
+    "                          \"Camille Corot (realism)\", \"Claude Monet (impressionism)\", \"Pablo Picasso (cubism)\", \"Gerhard Richter (abstract expressionism)\",\n",
+    "                          \"Hokusai (ukiyo-e)\", \"Gustav Klimt (art nouveau)\", \"Henri Matisse (abstract expressionism)\",\n",
+    "                          \"Walter Battiss\", \"Jackson Pollock\",  \"M.C. Escher\", \"Albert Gleizes\",  \"Wassily Kandinsky\",\n",
+    "                          \"Roy Lichtenstein\", \"Joan Miro\"\n",
+    "                          ],\n",
+    "                value=\"Andre Derain (fauvism)\",\n",
+    "                scale=1\n",
     "            )\n",
     "\n",
-    "        gr.on([text.submit, btn.click], demo_inference_gen, inputs=[adapter_choice, text, samples, seed, steps, scale], outputs=gallery)\n",
-    "        advanced_button.click(\n",
-    "            None,\n",
-    "            [],\n",
-    "            text,\n",
-    "        )\n",
+    "        with gr.Row():\n",
+    "            steps = gr.Slider(label=\"Steps\", minimum=1, maximum=50, value=20, step=1)\n",
+    "            adapter_scale = gr.Slider(label=\"Stylization Scale\", minimum=0, maximum=1.5, value=1., step=0.1, scale=1)\n",
+    "\n",
+    "        with gr.Row():\n",
+    "            seed = gr.Slider(label=\"Seed\",minimum=0,maximum=2147483647,step=1,randomize=True,scale=1)\n",
+    "\n",
+    "\n",
+    "        gr.on([btn_gen_ori.click], demo_inference_gen_ori, inputs=[text, seed, steps, scale], outputs=gallery_gen_ori)\n",
+    "        gr.on([btn_gen_art.click], demo_inference_gen_artistic, inputs=[adapter_choice, text, seed, steps, scale, adapter_scale], outputs=gallery_gen_art)\n",
     "\n",
+    "        gr.on([btn_style_ori.click], demo_inference_stylization_ori, inputs=[gallery_stylization_ref, text, seed, steps, scale, start_timestep], outputs=gallery_stylization_ori)\n",
+    "        gr.on([btn_style_art.click], demo_inference_stylization_artistic, inputs=[gallery_stylization_ref, adapter_choice, text, seed, steps, scale, adapter_scale, start_timestep], outputs=gallery_stylization_art)\n",
     "\n",
     "block.launch(share=True)"
    ]

utils/train_util.py

@@ -249,7 +249,8 @@ def get_noisy_image(
     image = img
     # im_orig = image
     device = vae.device
-    image = image_processor.preprocess(image).to(device)
+    weight_dtype = vae.dtype
+    image = image_processor.preprocess(image).to(device).to(weight_dtype).to(weight_dtype)
 
     init_latents = vae.encode(image).latent_dist.sample(None)
     init_latents = vae.config.scaling_factor * init_latents