build first

app.py

@@ -0,0 +1,59 @@
+import streamlit as st
+from functions import generate_controlnet_image, generate_image, generate_video
+from choices import text_to_image_models,controlnet_models,image_to_video_models
+
+def text_to_image(input_text, selected_model):
+
+    result = generate_image(selected_model,input_text)
+
+    return result
+
+
+def image_to_image(input_text,input_image, selected_model, selected_algorithm):
+
+
+    return "path_to_image_to_image_result.jpg"
+
+
+def image_to_video(input_text,input_image, selected_model1, selected_model2, selected_algorithm):
+    
+    
+    return "path_to_image_to_video_result.jpg"
+
+
+def main():
+    st.title("Image Processing Options")
+
+    
+    processing_option = st.selectbox("Select Processing Option", ["Text to Image", "Image to Image", "Image to Video"])
+
+    
+    if processing_option == "Text to Image":
+        input_text = st.text_area("Enter Text for Image")
+        selected_model = st.selectbox("Select Model", text_to_image_models)
+        if st.button("Generate Image"):
+            result_image_path = text_to_image(input_text, selected_model)
+            st.image(result_image_path, caption="Result Image", use_column_width=True)
+
+    elif processing_option == "Image to Image" or processing_option == "Image to Video":
+        input_text = st.text_area("Enter Text for Image")
+        input_image = st.file_uploader("Upload Input Image", type=["jpg", "png"])
+        selected_algorithm = st.selectbox("Select Algorithm", ["Algorithm 1", "Algorithm 2"])
+        
+        if processing_option == "Image to Image":
+            selected_model = st.selectbox("Select Model", controlnet_models)
+        elif processing_option == "Image to Video":
+            selected_model = st.selectbox("Select Model", image_to_video_models)
+
+        if st.button("Process"):
+            if input_image is not None:
+                if processing_option == "Image to Image":
+                    result_image_path = image_to_image(input_text,input_image, selected_model, selected_algorithm)
+                elif processing_option == "Image to Video":
+                    result_image_path = image_to_video(input_text,input_image, selected_model, selected_algorithm)
+                st.image(result_image_path, caption="Result Image", use_column_width=True)
+            else:
+                st.warning("Please upload an input image.")
+
+if __name__ == "__main__":
+    main()

choices.py

@@ -0,0 +1,5 @@
+text_to_image_models = ['stabilityai/stable-diffusion-xl-base-1.0']
+
+controlnet_models=['']
+
+image_to_video_models=['']

functions.py

@@ -0,0 +1,50 @@
+from diffusers import StableDiffusionPipeline,StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+from diffusers.utils import load_image
+import torch
+
+
+def generate_image(model_name,input_text):
+    pipe = StableDiffusionPipeline.from_pretrained(model_name, torch_dtype=torch.float16)
+    # pipe = pipe.to("cuda")
+
+    prompt = input_text
+    image = pipe(prompt).images[0]  
+        
+    image.save("testo.png")
+    return image
+
+def generate_controlnet_image(model_name,algorithm,input_image,input_text):
+    mask_image = generate_mask(input_image,algorithm)
+
+    base_model_path = model_name
+    controlnet_path = "lllyasviel/control_v11p_sd15_inpaint"
+
+    controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16)
+    pipe = StableDiffusionControlNetPipeline.from_pretrained(
+        base_model_path, controlnet=controlnet, torch_dtype=torch.float16
+    )
+
+    # speed up diffusion process with faster scheduler and memory optimization
+    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+    # remove following line if xformers is not installed or when using Torch 2.0.
+    pipe.enable_xformers_memory_efficient_attention()
+    # memory optimization.
+    pipe.enable_model_cpu_offload()
+
+    control_image = load_image(mask_image)
+    prompt = "pale golden rod circle with old lace background"
+
+    # generate image
+    generator = torch.manual_seed(0)
+    image = pipe(
+        prompt, num_inference_steps=20, generator=generator, image=control_image
+    ).images[0]
+    image.save("./output.png")
+    
+    return mask_image
+
+def generate_video(model_name,input_image,input_text):
+    return input_image
+
+def generate_mask(image,algorithm):
+    pass

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+diffusers 
+torch
+transformers
+opencv-python