Upload 14 files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+frontend/example/img_2.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/example/img_3.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/example/img_4.jpg filter=lfs diff=lfs merge=lfs -text
+frontend/example/output.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,12 @@
 ---
 title: Memory Carousel
-emoji: 🐠
-colorFrom: green
-colorTo: purple
+emoji: 🎞️ 
+colorFrom: purple
+colorTo: yellow
 sdk: gradio
-sdk_version: 4.19.1
-app_file: app.py
+sdk_version: 3.34.0
+python_version: 3.11.3
+app_file: frontend/app.py
 pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+license: apache-2.0
+---

backend/main.py

@@ -0,0 +1,23 @@
+from fastapi import FastAPI, HTTPException
+from models import ImageInpaintingRequest, ImageInpaintingResponse
+from utils import process_images_and_inpaint
+from typing import List
+import uvicorn
+
+app = FastAPI()
+
+@app.get("/")
+def hello():
+    return {"message":"Test FastAPI"}
+
+@app.post("/inpaint", response_model=ImageInpaintingResponse)
+async def inpaint_images(request: ImageInpaintingRequest):
+    try:
+        inpainted_image_b64 = process_images_and_inpaint(request.images, request.alpha_gradient_width, request.init_image_height)
+        return ImageInpaintingResponse(inpainted_image=inpainted_image_b64)
+    except Exception as e:
+        print(e)
+        raise HTTPException(status_code=500, detail=str(e))
+    
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

backend/models.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class ImageInpaintingRequest(BaseModel):
+    images: List[str]
+    alpha_gradient_width: int
+    init_image_height: int
+    
+    class Config:
+        arbitrary_types_allowed = True
+
+class ImageInpaintingResponse(BaseModel):
+    inpainted_image: str
+
+    class Config:
+        arbitrary_types_allowed = True

backend/utils.py

@@ -0,0 +1,120 @@
+import base64
+import requests
+from json import dumps, dump
+from PIL import Image
+from io import BytesIO
+import time
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+endpoint = 'https://serving.hopter.staging.picc.co/api/v1/services/gen-ai-image-expansion/predictions'
+token = os.getenv('API_TOKEN')
+
+def pil_to_b64(image:Image.Image) -> str:
+    buffered = BytesIO()
+    image.save(buffered, format="PNG", quality=80)
+    img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    prefix = 'data:image/png;base64,'
+    return prefix + img_str
+
+def b64_to_pil(b64_string):
+    # Remove the Base64 prefix if present
+    if b64_string.startswith('data:image'):
+        b64_string = b64_string.split(';base64,', 1)[1]
+    # Decode the Base64 string to bytes
+    image_bytes = base64.b64decode(b64_string)
+    # Create a BytesIO object and load the image bytes
+    image_buffer = BytesIO(image_bytes)
+    image = Image.open(image_buffer)
+    return image
+
+def resize_image(image, max_height=768):
+    scale = max_height/image.height
+    return image.resize((int(image.width * scale), int(image.height * scale)))
+
+def prepare_init_image_mask(images: [Image.Image], alpha_gradient_width=80, init_image_height=768):  # type: ignore
+    total_width = sum([ im.width for im in images])
+    init_image = Image.new('RGBA', (total_width,init_image_height))
+    
+    # Paste input images on init_image
+    x_coord = 0
+    for im in images: 
+        init_image.paste(im, (x_coord, 0))
+        x_coord += im.width
+    
+    # Add linear alpha gradient  
+    x_coord = 0
+    is_right_patch = True
+    i = 0
+    while i <= len(images) - 1:
+        im = images[i]
+        if i == len(images) - 1 and is_right_patch: 
+            break 
+        if is_right_patch:
+            alpha = Image.linear_gradient('L').rotate(-90).resize((alpha_gradient_width, init_image_height))
+            tmp_img = init_image.crop((x_coord+im.width - alpha_gradient_width, 0, x_coord+im.width, init_image_height))
+            tmp_img.putalpha(alpha)
+            init_image.paste(tmp_img, (x_coord+im.width - alpha_gradient_width, 0))
+            x_coord += im.width
+            i += 1
+            is_right_patch = False
+        else: 
+            alpha = Image.linear_gradient('L').rotate(90).resize((alpha_gradient_width, init_image_height))
+            tmp_img = init_image.crop((x_coord, 0, x_coord+alpha_gradient_width, init_image_height))
+            tmp_img.putalpha(alpha)
+            init_image.paste(tmp_img, (x_coord, 0))
+            is_right_patch = True
+    
+    # Generate inpainting mask 
+    mask = Image.new('RGBA', (total_width, init_image_height), (0, 0, 0))
+    x_coord = 0
+    for im in images[:-1]: 
+        mask_patch = Image.new('RGBA', (alpha_gradient_width*2, init_image_height), (255, 255, 255))
+        mask.paste(mask_patch, (x_coord + im.width - alpha_gradient_width, 0))
+        x_coord += im.width 
+        
+    
+    # Crop init_image and mask into batches
+    x_coord = 0 
+    init_image_mask_pair = []
+    init_image_patch_x_coord = []
+    
+    for im in images[:-1]:
+        crop_start_x = x_coord + im.width - init_image_height // 2
+        crop_end_x = x_coord + im.width + init_image_height // 2
+        tmp_img = init_image.crop((crop_start_x, 0, min(total_width, crop_end_x), init_image_height))
+        tmp_mask = mask.crop((crop_start_x, 0, min(total_width, crop_end_x), init_image_height))
+        init_image_mask_pair.append((tmp_img, tmp_mask))
+        init_image_patch_x_coord.append(crop_start_x)
+        x_coord += im.width
+    return init_image, mask, init_image_mask_pair, init_image_patch_x_coord
+
+def attach_images_with_loc(inpainted_results, init_image_patch_x_coord, full_init_img):
+    full_init_img = full_init_img
+    for im, loc in zip(inpainted_results, init_image_patch_x_coord):
+        full_init_img.paste(im, (loc, 0))
+    return full_init_img
+
+
+def inpainting_api_call(input_image, input_mask, token, endpoint):
+    body = {
+        "input": {
+            "initial_image_b64": pil_to_b64(input_image),
+            "mask_image_b64": pil_to_b64(input_mask.convert('L'))
+      }
+    }
+
+    json_data = dumps(body)
+    start = time.time()
+    resp_inpaint = requests.post(endpoint, data=json_data, headers={"Authorization": f"Bearer {token}"})
+    print(f"Execution time: {time.time() - start}")
+    return b64_to_pil(resp_inpaint.json()['output']['inpainted_image_b64'])
+
+def process_images_and_inpaint(images, alpha_gradient_width=100, init_image_height=768):
+    images = [ resize_image(b64_to_pil(im)).convert("RGBA") for im in images ]
+    full_init_img, full_mask, init_image_mask_pair, init_image_patch_x_coord = prepare_init_image_mask(images, alpha_gradient_width, init_image_height)
+    results = [ inpainting_api_call(im, mask, token, endpoint) for im, mask in init_image_mask_pair]
+    attached_image = pil_to_b64(attach_images_with_loc(results, init_image_patch_x_coord, full_init_img))
+    return attached_image
+ 

frontend/app.py

@@ -0,0 +1,71 @@
+import gradio as gr
+import requests
+from utils import resize_image, pil_to_b64, b64_to_pil, process_images_and_inpaint
+
+USE_FASTAPI = False
+FAST_API_ENDPOINT = 'http://127.0.0.1:5000/inpaint'
+
+def run_inpainting(img_1, img_2, img_3, img_4, alpha_gradient_width, init_image_height, USE_FASTAPI):
+    images = []
+    for img in [img_1, img_2, img_3, img_4]:
+        if img is not None: 
+            images.append(pil_to_b64(resize_image(img, init_image_height)))
+    if USE_FASTAPI:
+        return call_inpainting_api(img_1, img_2, img_3, img_4, alpha_gradient_width, init_image_height)
+    else: 
+        return process_images_and_inpaint(images, alpha_gradient_width, init_image_height)
+
+def call_inpainting_api(img_1, img_2, img_3, img_4, alpha_gradient_width, init_image_height):
+    images = []
+    for img in [img_1, img_2, img_3, img_4]:
+        if img is not None: 
+            images.append(pil_to_b64(resize_image(img, init_image_height)))
+    response = requests.post(FAST_API_ENDPOINT, json={
+        "images": images, 
+        "alpha_gradient_width": alpha_gradient_width,
+        "init_image_height": init_image_height
+    })
+    if response.status_code == 200:
+        return b64_to_pil(response.json()["inpainted_image"])
+    else:
+        return "Error calling inpainting API"
+
+TITLE = """<h2 align="center"> 🎞️ Memory Carousel </h2>"""
+
+# Define the Gradio interface 
+with gr.Blocks() as demo:
+    gr.HTML(TITLE)
+    with gr.Column():
+        with gr.Row():
+            input_image_1 = gr.Image(type='pil', label="First image")
+            input_image_2 = gr.Image(type='pil', label="Second image")
+        with gr.Row():
+            input_image_3 = gr.Image(type='pil', label="Third image(optional)")
+            input_image_4 = gr.Image(type='pil', label="Fourth image(optional)")
+        with gr.Row():
+            alpha_gradient_width = gr.Number(value=100, label="Alpha Gradient Width")
+            init_image_height = gr.Number(value=768, label="Init Image Height")
+        generate_button = gr.Button("Generate")
+        output = gr.Image(type='pil')
+
+        example_list = gr.Examples(
+        examples=[['./example/img_1.jpg', './example/img_2.jpg', './example/img_3.jpg', './example/img_4.jpg', 100, 768]],
+            inputs=[
+                input_image_1, 
+                input_image_2, 
+                input_image_3, 
+                input_image_4,
+                alpha_gradient_width, 
+                init_image_height
+                ],
+            outputs=[output],
+            fn=call_inpainting_api,
+            cache_examples=True,
+        )
+
+        generate_button.click(
+            fn=call_inpainting_api, 
+            inputs=[input_image_1, input_image_2, input_image_3, input_image_4, alpha_gradient_width, init_image_height], 
+            outputs=[output]
+        )
+demo.launch()

frontend/utils.py

@@ -0,0 +1,119 @@
+import base64
+import requests
+from json import dumps, dump
+from PIL import Image
+from io import BytesIO
+import time
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+endpoint = 'https://serving.hopter.staging.picc.co/api/v1/services/gen-ai-image-expansion/predictions'
+token = os.getenv('API_TOKEN')
+
+def pil_to_b64(image:Image.Image) -> str:
+    buffered = BytesIO()
+    image.save(buffered, format="PNG", quality=80)
+    img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    prefix = 'data:image/png;base64,'
+    return prefix + img_str
+
+def b64_to_pil(b64_string):
+    # Remove the Base64 prefix if present
+    if b64_string.startswith('data:image'):
+        b64_string = b64_string.split(';base64,', 1)[1]
+    # Decode the Base64 string to bytes
+    image_bytes = base64.b64decode(b64_string)
+    # Create a BytesIO object and load the image bytes
+    image_buffer = BytesIO(image_bytes)
+    image = Image.open(image_buffer)
+    return image
+
+def resize_image(image, max_height=768):
+    scale = max_height/image.height
+    return image.resize((int(image.width * scale), int(image.height * scale)))
+
+def prepare_init_image_mask(images: [Image.Image], alpha_gradient_width=80, init_image_height=768):  # type: ignore
+    total_width = sum([ im.width for im in images])
+    init_image = Image.new('RGBA', (total_width,init_image_height))
+    
+    # Paste input images on init_image
+    x_coord = 0
+    for im in images: 
+        init_image.paste(im, (x_coord, 0))
+        x_coord += im.width
+    
+    # Add linear alpha gradient  
+    x_coord = 0
+    is_right_patch = True
+    i = 0
+    while i <= len(images) - 1:
+        im = images[i]
+        if i == len(images) - 1 and is_right_patch: 
+            break 
+        if is_right_patch:
+            alpha = Image.linear_gradient('L').rotate(-90).resize((alpha_gradient_width, init_image_height))
+            tmp_img = init_image.crop((x_coord+im.width - alpha_gradient_width, 0, x_coord+im.width, init_image_height))
+            tmp_img.putalpha(alpha)
+            init_image.paste(tmp_img, (x_coord+im.width - alpha_gradient_width, 0))
+            x_coord += im.width
+            i += 1
+            is_right_patch = False
+        else: 
+            alpha = Image.linear_gradient('L').rotate(90).resize((alpha_gradient_width, init_image_height))
+            tmp_img = init_image.crop((x_coord, 0, x_coord+alpha_gradient_width, init_image_height))
+            tmp_img.putalpha(alpha)
+            init_image.paste(tmp_img, (x_coord, 0))
+            is_right_patch = True
+    
+    # Generate inpainting mask 
+    mask = Image.new('RGBA', (total_width, init_image_height), (0, 0, 0))
+    x_coord = 0
+    for im in images[:-1]: 
+        mask_patch = Image.new('RGBA', (alpha_gradient_width*2, init_image_height), (255, 255, 255))
+        mask.paste(mask_patch, (x_coord + im.width - alpha_gradient_width, 0))
+        x_coord += im.width 
+        
+    
+    # Crop init_image and mask into batches
+    x_coord = 0 
+    init_image_mask_pair = []
+    init_image_patch_x_coord = []
+    
+    for im in images[:-1]:
+        crop_start_x = x_coord + im.width - init_image_height // 2
+        crop_end_x = x_coord + im.width + init_image_height // 2
+        tmp_img = init_image.crop((crop_start_x, 0, min(total_width, crop_end_x), init_image_height))
+        tmp_mask = mask.crop((crop_start_x, 0, min(total_width, crop_end_x), init_image_height))
+        init_image_mask_pair.append((tmp_img, tmp_mask))
+        init_image_patch_x_coord.append(crop_start_x)
+        x_coord += im.width
+    return init_image, mask, init_image_mask_pair, init_image_patch_x_coord
+
+def attach_images_with_loc(inpainted_results, init_image_patch_x_coord, full_init_img):
+    full_init_img = full_init_img
+    for im, loc in zip(inpainted_results, init_image_patch_x_coord):
+        full_init_img.paste(im, (loc, 0))
+    return full_init_img
+
+
+def inpainting_api_call(input_image, input_mask, token, endpoint):
+    body = {
+        "input": {
+            "initial_image_b64": pil_to_b64(input_image),
+            "mask_image_b64": pil_to_b64(input_mask.convert('L'))
+      }
+    }
+
+    json_data = dumps(body)
+    start = time.time()
+    resp_inpaint = requests.post(endpoint, data=json_data, headers={"Authorization": f"Bearer {token}"})
+    print(f"Execution time: {time.time() - start}")
+    return b64_to_pil(resp_inpaint.json()['output']['inpainted_image_b64'])
+
+def process_images_and_inpaint(images, alpha_gradient_width=100, init_image_height=768):
+    images = [ resize_image(b64_to_pil(im)).convert("RGBA") for im in images ]
+    full_init_img, full_mask, init_image_mask_pair, init_image_patch_x_coord = prepare_init_image_mask(images, alpha_gradient_width, init_image_height)
+    results = [ inpainting_api_call(im, mask, token, endpoint) for im, mask in init_image_mask_pair]
+    attached_image = pil_to_b64(attach_images_with_loc(results, init_image_patch_x_coord, full_init_img))
+    return attached_image

requirements.txt

@@ -0,0 +1,8 @@
+fastapi
+uvicorn
+python-multipart
+Pillow
+requests
+python-dotenv
+gradio
+gunicorn

start_server.sh

@@ -0,0 +1,2 @@
+cd backend
+uvicorn --port 5000 --host 127.0.0.1 main:app --reload

start_web_app.sh

@@ -0,0 +1,2 @@
+ cd frontend
+ python app.py