up

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Editor-based HTTP Client requests
-/httpRequests/
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml

.idea/RemoveFurnitureV1.iml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -1,15 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0">
-    <option name="myName" value="Project Default" />
-    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="ignoredPackages">
-        <value>
-          <list size="2">
-            <item index="0" class="java.lang.String" itemvalue="opencv_python" />
-            <item index="1" class="java.lang.String" itemvalue="skimage" />
-          </list>
-        </value>
-      </option>
-    </inspection_tool>
-  </profile>
-</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="Black">
-    <option name="sdkName" value="Python 3.12" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12" project-jdk-type="Python SDK" />
-</project>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/RemoveFurnitureV1.iml" filepath="$PROJECT_DIR$/.idea/RemoveFurnitureV1.iml" />
-    </modules>
-  </component>
-</project>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="" vcs="Git" />
-  </component>
-</project>

app.py

@@ -1,22 +1,9 @@
 import gradio as gr
-import io
-from PIL import Image
 import numpy as np
+from models import make_inpainting
+import utils
 
-# from config import setResoluton
-from models import make_image_controlnet, make_inpainting
-from preprocessing import get_mask
-
-def image_to_byte_array(image: Image) -> bytes:
-    # BytesIO is a fake file stored in memory
-    imgByteArr = io.BytesIO()
-    # image.save expects a file as a argument, passing a bytes io ins
-    image.save(imgByteArr, format='png')  # image.format
-    # Turn the BytesIO object back into a bytes object
-    imgByteArr = imgByteArr.getvalue()
-    return imgByteArr
-
-def predict(input_img1,
+def removeFurniture(input_img1,
             input_img2,
             positive_prompt,
             negative_prompt,
@@ -24,24 +11,15 @@ def predict(input_img1,
             resolution
             ):
 
-
-
-    print("predict")
-    # bla bla
-    # input_img1 = Image.fromarray(input_img1)
-    # input_img2 = Image.fromarray(input_img2)
-
-    # setResoluton(resolution)
+    print("removeFurniture")
     HEIGHT = resolution
     WIDTH = resolution
-    # WIDTH = resolution
-    # HEIGHT = resolution
 
     input_img1 = input_img1.resize((resolution, resolution))
     input_img2 = input_img2.resize((resolution, resolution))
 
     canvas_mask = np.array(input_img2)
-    mask = get_mask(canvas_mask)
+    mask = utils.get_mask(canvas_mask)
 
     print(input_img1, mask, positive_prompt, negative_prompt)
 
@@ -58,41 +36,37 @@ def predict(input_img1,
 
     return retList
 
-
-app = gr.Interface(
-    predict,
-    inputs=[gr.Image(label="img", sources=['upload'], type="pil"),
-            gr.Image(label="mask", sources=['upload'], type="pil"),
-            gr.Textbox(label="positive_prompt",value="empty room"),
-            gr.Textbox(label="negative_prompt",value=""),
-            gr.Number(label="num_of_images",value=2),
-            gr.Number(label="resolution",value=512)
-            ],
-    outputs= [
-        gr.Image(label="resp0"),
-        gr.Image(label="resp1"),
-        gr.Image(label="resp2"),
-        gr.Image(label="resp3"),
-        gr.Image(label="resp4"),
-        gr.Image(label="resp5"),
-        gr.Image(label="resp6"),
-        gr.Image(label="resp7"),
-        gr.Image(label="resp8"),
-        gr.Image(label="resp9")],
-    title="rem fur 1",
-)
-
-app.launch(share=True)
-
-#
-
-
-# gr.Interface(
-#     test1,
-#     inputs=[gr.Textbox(label="param1")],
-#     outputs= gr.Textbox(label="result"),
-#     title="rem fur 1",
-# ).launch(share=True)
-
-
-
+def segmentation(image):
+    return image
+
+def upscale(image):
+    return image
+
+with gr.Blocks() as app:    
+    with gr.Row():
+
+        gr.Button("FurnituRemove").click(removeFurniture, 
+                                        inputs=[gr.Image(label="img", type="pil"),
+                                                gr.Image(label="mask", type="pil"),
+                                                gr.Textbox(label="positive_prompt",value="empty room"),
+                                                gr.Textbox(label="negative_prompt",value=""),
+                                                gr.Number(label="num_of_images",value=2),
+                                                gr.Number(label="resolution",value=512)
+                                                ], 
+                                        outputs=[
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image(),
+                                                gr.Image()])
+
+        gr.Button("Segmentation").click(segmentation, inputs=gr.Image(type="pil"), outputs=gr.Image())
+
+        gr.Button("Upscale").click(upscale, inputs=gr.Image(type="pil"), outputs=gr.Image())
+
+app.launch(debug=True)

explanation.py

@@ -1,51 +0,0 @@
-import streamlit as st
-
-def make_inpainting_explanation():
-    with st.expander("Explanation inpainting", expanded=False):
-        st.write("In the inpainting mode, you can draw regions on the input image that you want to regenerate. "
-                "This can be useful to remove unwanted objects from the image or to improve the consistency of the image."
-                )
-        st.image("content/inpainting_sidebar.png", caption="Image before inpainting, note the ornaments on the wall", width=500)
-        st.write("You can find drawing options in the sidebar. There are two modes: freedraw and polygon. Freedraw allows the user to draw with a pencil of a certain width. "
-                 "Polygon allows the user to draw a polygon by clicking on the image to add a point. The polygon is closed by right clicking.")
-
-        st.write("### Example inpainting")
-        st.write("In the example below, the ornaments on the wall are removed. The inpainting is done by drawing a mask on the image.")
-        st.image("content/inpainting_before.jpg", caption="Image before inpainting, note the ornaments on the wall")
-        st.image("content/inpainting_after.png", caption="Image before inpainting, note the ornaments on the wall")
-        
-def make_regeneration_explanation():
-    with st.expander("Explanation object regeneration"):
-        st.write("In this object regeneration mode, the model calculates which objects occur in the image. "
-                 "The user can then select which objects can be regenerated by the controlnet model by adding them in the multiselect box. "
-                 "All the object classes that are not selected will remain the same as in the original image."
-                 )
-        st.write("### Example object regeneration")
-        st.write("In the example below, the room consists of various objects such as wall, ceiling, floor, lamp, bed, ... "
-                 "In the multiselect box, all the objects except for 'lamp', 'bed and 'table' are selected to be regenerated. "
-                 )
-        st.image("content/regen_example.png", caption="Room where all concepts except for 'bed', 'lamp', 'table' are regenerated")
-
-def make_segmentation_explanation():
-    with st.expander("Segmentation mode", expanded=False):
-        st.write("In the segmentation mode, the user can use his imagination and the paint brush to place concepts in the image. "
-                 "In the left sidebar, you can first find the high level category of the concept you want to add, such as 'lighting', 'floor', .. "
-                 "After selecting the category, you can select the specific concept you want to add in the 'Choose a color' dropdown. "
-                 "This will change the color of the paint brush, which you can then use to draw on the input image. "
-                 "The model will then regenerate the image with the concepts you have drawn and leave the rest of the image unchanged. "
-                 )
-        st.image("content/sidebar segmentation.png", caption="Sidebar with segmentation options", width=300)
-        st.write("You can choose the freedraw mode which gives you a pencil of a certain (chosen) width or the polygon mode. With the polygon mode you can click to add a point to the polygon and close the polygon by right clicking. ")
-        st.write("Important: "
-                 "it's not easy to draw a good segmentation mask. This is because you need to keep in mind the perspective of the room and the exact "
-                 "shape of the object you want to draw within this perspective. Controlnet will follow your segmentation mask pretty well, so "
-                 "a non-natural object shape will sometimes result in weird outputs. However, give it a try and see what you can do! "
-                 )
-        st.image("content/segmentation window.png", caption="Example of a segmentation mask drawn on the input image to add a window to the room")
-        st.write("Tip: ")
-        st.write("In the concepts dropdown, you can select 'keep background' (which is a white color). Everything drawn in this color will use "
-                 "the original underlying segmentation mask. This can be useful to help with generating other objects, since you give the model a some "
-                 "freedom to generate outside the object borders."
-                 )
-        st.image("content/keep background 1.png", caption="Image with a poster drawn on the wall.")
-        st.image("content/keep background 2.png", caption="Image with a poster drawn on the wall surrounded by 'keep background'.")

models.py

@@ -2,17 +2,13 @@
 import logging
 from typing import List, Tuple, Dict
 
-import streamlit as st
+
 import torch
-import gc
-import time
 import numpy as np
 from PIL import Image
-from PIL import ImageFilter
 
 from diffusers import ControlNetModel, UniPCMultistepScheduler
 
-# from config import WIDTH, HEIGHT
 from palette import ade_palette
 from stable_diffusion_controlnet_inpaint_img2img import StableDiffusionControlNetInpaintImg2ImgPipeline
 from helpers import flush, postprocess_image_masking, convolution
@@ -48,7 +44,7 @@ def make_image_controlnet(image: np.ndarray,
     mask_image_postproc = convolution(mask_image)
 
 
-    st.success(f"{pipe.queue_size} images in the queue, can take up to {(pipe.queue_size+1) * 10} seconds")
+    
     generated_image = pipe(
         prompt=positive_prompt,
         negative_prompt=negative_prompt,
@@ -90,7 +86,7 @@ def make_inpainting(positive_prompt: str,
     flush()
     retList=[]
     for x in range(num_of_images):
-        st.success(f"{pipe.queue_size} images in the queue, can take up to {(pipe.queue_size+1) * 10} seconds")
+        
         resp = pipe(image=image,
                         mask_image=mask_image,
                         prompt=positive_prompt,

pipelines.py

@@ -1,7 +1,7 @@
 import logging
 from typing import List, Tuple, Dict
 
-import streamlit as st
+
 import torch
 import gc
 import time
@@ -105,7 +105,7 @@ class SDPipeline:
 
 
 
-@st.cache_resource(max_entries=5)
+
 def get_controlnet():
     """Method to load the controlnet model
     Returns:
@@ -116,7 +116,6 @@ def get_controlnet():
 
 
 
-@st.cache_resource(max_entries=5)
 def get_inpainting_pipeline():
     """Method to load the inpainting pipeline
     Returns:

preprocessing.py

@@ -4,7 +4,7 @@ from typing import List, Tuple
 
 import numpy as np
 from PIL import Image, ImageFilter
-import streamlit as st
+
 
 from config import COLOR_RGB
 # from enhance_config import ENHANCE_SETTINGS
@@ -51,85 +51,5 @@ def preprocess_seg_mask(canvas_seg, real_seg: Image.Image = None) -> Tuple[np.nd
     return mask, image_seg
 
 
-def get_mask(image_mask: np.ndarray) -> np.ndarray:
-    """Get the mask from the segmentation mask.
-    Args:
-        image_mask (np.ndarray): segmentation mask
-    Returns:
-        np.ndarray: mask
-    """
-    # average the colors of the segmentation masks
-    average_color = np.mean(image_mask, axis=(2))
-    mask = average_color[:, :] > 0
-    if mask.sum() > 0:
-        mask = mask * 1
-    return mask
-
-
-# def get_image() -> np.ndarray:
-#
-#     """Get the image from the session state.
-#     Returns:
-#         np.ndarray: image
-#     """
-#     if 'initial_image' in st.session_state and st.session_state['initial_image'] is not None:
-#         initial_image = st.session_state['initial_image']
-#         if isinstance(initial_image, Image.Image):
-#             return np.array(initial_image.resize((WIDTH, HEIGHT)))
-#         else:
-#             return np.array(Image.fromarray(initial_image).resize((WIDTH, HEIGHT)))
-#     else:
-#         return None
-
-
-# def make_enhance_config(segmentation, objects=None):
-    """Make the enhance config for the segmentation image.
-    """
-    info = ENHANCE_SETTINGS[objects]
-
-    segmentation = np.array(segmentation)
-
-    if 'replace' in info:
-        replace_color = info['replace']
-        mask = np.zeros(segmentation.shape)
-        for color in info['colors']:
-            mask[np.all(segmentation == color, axis=-1)] = [1, 1, 1]
-            segmentation[np.all(segmentation == color, axis=-1)] = replace_color
-
-    if info['inverse'] is False:
-        mask = np.zeros(segmentation.shape)
-        for color in info['colors']:
-            mask[np.all(segmentation == color, axis=-1)] = [1, 1, 1]
-    else:
-        mask = np.ones(segmentation.shape)
-        for color in info['colors']:
-            mask[np.all(segmentation == color, axis=-1)] = [0, 0, 0]
-
-    st.session_state['positive_prompt'] = info['positive_prompt']
-    st.session_state['negative_prompt'] = info['negative_prompt']
-
-    if info['inpainting'] is True:
-        mask = mask.astype(np.uint8)
-        mask = Image.fromarray(mask)
-        mask = mask.filter(ImageFilter.GaussianBlur(radius=13))
-        mask = mask.filter(ImageFilter.MaxFilter(size=9))
-        mask = np.array(mask)
-
-        mask[mask < 0.1] = 0
-        mask[mask >= 0.1] = 1
-        mask = mask.astype(np.uint8)
-
-        conditioning = dict(
-            mask_image=mask,
-            positive_prompt=info['positive_prompt'],
-            negative_prompt=info['negative_prompt'],
-        )
-    else:
-        conditioning = dict(
-            mask_image=mask,
-            controlnet_conditioning_image=segmentation,
-            positive_prompt=info['positive_prompt'],
-            negative_prompt=info['negative_prompt'],
-            strength=info['strength']
-        )
-    return conditioning, info['inpainting']
+
+

segmentation.py

@@ -1,7 +1,6 @@
 import logging
 from typing import List, Tuple, Dict
 
-import streamlit as st
 import torch
 import gc
 import numpy as np
@@ -18,7 +17,7 @@ def flush():
     gc.collect()
     torch.cuda.empty_cache()
 
-@st.cache_resource(max_entries=5)
+
 def get_segmentation_pipeline() -> Tuple[AutoImageProcessor, UperNetForSemanticSegmentation]:
     """Method to load the segmentation pipeline
     Returns:

utils.py

@@ -0,0 +1,27 @@
+
+import numpy as np
+from PIL import Image
+
+def image_to_byte_array(image: Image) -> bytes:
+    # BytesIO is a fake file stored in memory
+    imgByteArr = io.BytesIO()
+    # image.save expects a file as a argument, passing a bytes io ins
+    image.save(imgByteArr, format='png')  # image.format
+    # Turn the BytesIO object back into a bytes object
+    imgByteArr = imgByteArr.getvalue()
+    return imgByteArr
+
+
+def get_mask(image_mask: np.ndarray) -> np.ndarray:
+    """Get the mask from the segmentation mask.
+    Args:
+        image_mask (np.ndarray): segmentation mask
+    Returns:
+        np.ndarray: mask
+    """
+    # average the colors of the segmentation masks
+    average_color = np.mean(image_mask, axis=(2))
+    mask = average_color[:, :] > 0
+    if mask.sum() > 0:
+        mask = mask * 1
+    return mask