DSSCE_HW4 / app.py
Morris0401's picture
Update app.py
e235f29 verified
import os
print(os.path.abspath(os.getcwd()))
print(os.listdir())
whl_path = '/home/user/app/wheels/dlib-19.24.6-cp310-cp310-linux_x86_64.whl'
print(f"檢查檔案是否存在: {os.path.exists(whl_path)}")
os.system("python -m pip install dlib")
os.system("pip install --upgrade ./wheels/opencv_python_headless-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
import gradio as gr
import cv2
import numpy as np
import random
import sys
import os
import dlib
import glob
from skimage import io
from imutils import face_utils
import math
from subprocess import Popen, PIPE
from PIL import Image
import subprocess
import argparse
import shutil
# Check if a point is inside a rectangle
def rect_contains(rect, point):
if point[0] < rect[0]:
return False
elif point[1] < rect[1]:
return False
elif point[0] > rect[2]:
return False
elif point[1] > rect[3]:
return False
return True
# Write the delaunay triangles into a file
def draw_delaunay(f_w, f_h, subdiv, dictionary1):
list4 = []
triangleList = subdiv.getTriangleList()
r = (0, 0, f_w, f_h)
for t in triangleList :
pt1 = (int(t[0]), int(t[1]))
pt2 = (int(t[2]), int(t[3]))
pt3 = (int(t[4]), int(t[5]))
if rect_contains(r, pt1) and rect_contains(r, pt2) and rect_contains(r, pt3) :
list4.append((dictionary1[pt1],dictionary1[pt2],dictionary1[pt3]))
dictionary1 = {}
return list4
def make_delaunay(f_w, f_h, theList, img1, img2):
# Make a rectangle.
rect = (0, 0, f_w, f_h)
# Create an instance of Subdiv2D.
subdiv = cv2.Subdiv2D(rect)
# Make a points list and a searchable dictionary.
theList = theList.tolist()
points = [(int(x[0]),int(x[1])) for x in theList]
dictionary = {x[0]:x[1] for x in list(zip(points, range(76)))}
# Insert points into subdiv
for p in points :
subdiv.insert(p)
# Make a delaunay triangulation list.
list4 = draw_delaunay(f_w, f_h, subdiv, dictionary)
# Return the list.
return list4
class NoFaceFound(Exception):
"""Raised when there is no face found"""
pass
def calculate_margin_help(img1,img2):
size1 = img1.shape
size2 = img2.shape
diff0 = abs(size1[0]-size2[0])//2
diff1 = abs(size1[1]-size2[1])//2
avg0 = (size1[0]+size2[0])//2
avg1 = (size1[1]+size2[1])//2
return [size1,size2,diff0,diff1,avg0,avg1]
def crop_image(img1,img2):
[size1,size2,diff0,diff1,avg0,avg1] = calculate_margin_help(img1,img2)
if(size1[0] == size2[0] and size1[1] == size2[1]):
return [img1,img2]
elif(size1[0] <= size2[0] and size1[1] <= size2[1]):
scale0 = size1[0]/size2[0]
scale1 = size1[1]/size2[1]
if(scale0 > scale1):
res = cv2.resize(img2,None,fx=scale0,fy=scale0,interpolation=cv2.INTER_AREA)
else:
res = cv2.resize(img2,None,fx=scale1,fy=scale1,interpolation=cv2.INTER_AREA)
return crop_image_help(img1,res)
elif(size1[0] >= size2[0] and size1[1] >= size2[1]):
scale0 = size2[0]/size1[0]
scale1 = size2[1]/size1[1]
if(scale0 > scale1):
res = cv2.resize(img1,None,fx=scale0,fy=scale0,interpolation=cv2.INTER_AREA)
else:
res = cv2.resize(img1,None,fx=scale1,fy=scale1,interpolation=cv2.INTER_AREA)
return crop_image_help(res,img2)
elif(size1[0] >= size2[0] and size1[1] <= size2[1]):
return [img1[diff0:avg0,:],img2[:,-diff1:avg1]]
else:
return [img1[:,diff1:avg1],img2[-diff0:avg0,:]]
def crop_image_help(img1,img2):
[size1,size2,diff0,diff1,avg0,avg1] = calculate_margin_help(img1,img2)
if(size1[0] == size2[0] and size1[1] == size2[1]):
return [img1,img2]
elif(size1[0] <= size2[0] and size1[1] <= size2[1]):
return [img1,img2[-diff0:avg0,-diff1:avg1]]
elif(size1[0] >= size2[0] and size1[1] >= size2[1]):
return [img1[diff0:avg0,diff1:avg1],img2]
elif(size1[0] >= size2[0] and size1[1] <= size2[1]):
return [img1[diff0:avg0,:],img2[:,-diff1:avg1]]
else:
return [img1[:,diff1:avg1],img2[diff0:avg0,:]]
def generate_face_correspondences(theImage1, theImage2):
# Detect the points of face.
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
corresp = np.zeros((68,2))
imgList = crop_image(theImage1,theImage2)
list1 = []
list2 = []
j = 1
for img in imgList:
size = (img.shape[0],img.shape[1])
if(j == 1):
currList = list1
else:
currList = list2
# Ask the detector to find the bounding boxes of each face. The 1 in the
# second argument indicates that we should upsample the image 1 time. This
# will make everything bigger and allow us to detect more faces.
dets = detector(img, 1)
try:
if len(dets) == 0:
raise NoFaceFound
except NoFaceFound:
print("Sorry, but I couldn't find a face in the image.")
j=j+1
for k, rect in enumerate(dets):
# Get the landmarks/parts for the face in rect.
shape = predictor(img, rect)
# corresp = face_utils.shape_to_np(shape)
for i in range(0,68):
x = shape.part(i).x
y = shape.part(i).y
currList.append((x, y))
corresp[i][0] += x
corresp[i][1] += y
# cv2.circle(img, (x, y), 2, (0, 255, 0), 2)
# Add back the background
currList.append((1,1))
currList.append((size[1]-1,1))
currList.append(((size[1]-1)//2,1))
currList.append((1,size[0]-1))
currList.append((1,(size[0]-1)//2))
currList.append(((size[1]-1)//2,size[0]-1))
currList.append((size[1]-1,size[0]-1))
currList.append(((size[1]-1),(size[0]-1)//2))
# Add back the background
narray = corresp/2
narray = np.append(narray,[[1,1]],axis=0)
narray = np.append(narray,[[size[1]-1,1]],axis=0)
narray = np.append(narray,[[(size[1]-1)//2,1]],axis=0)
narray = np.append(narray,[[1,size[0]-1]],axis=0)
narray = np.append(narray,[[1,(size[0]-1)//2]],axis=0)
narray = np.append(narray,[[(size[1]-1)//2,size[0]-1]],axis=0)
narray = np.append(narray,[[size[1]-1,size[0]-1]],axis=0)
narray = np.append(narray,[[(size[1]-1),(size[0]-1)//2]],axis=0)
return [size,imgList[0],imgList[1],list1,list2,narray]
# Apply affine transform calculated using srcTri and dstTri to src and
# output an image of size.
def apply_affine_transform(src, srcTri, dstTri, size) :
# Given a pair of triangles, find the affine transform.
warpMat = cv2.getAffineTransform(np.float32(srcTri), np.float32(dstTri))
# Apply the Affine Transform just found to the src image
dst = cv2.warpAffine(src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101)
return dst
# Warps and alpha blends triangular regions from img1 and img2 to img
def morph_triangle(img1, img2, img, t1, t2, t, alpha) :
# Find bounding rectangle for each triangle
r1 = cv2.boundingRect(np.float32([t1]))
r2 = cv2.boundingRect(np.float32([t2]))
r = cv2.boundingRect(np.float32([t]))
# Offset points by left top corner of the respective rectangles
t1Rect = []
t2Rect = []
tRect = []
for i in range(0, 3):
tRect.append(((t[i][0] - r[0]),(t[i][1] - r[1])))
t1Rect.append(((t1[i][0] - r1[0]),(t1[i][1] - r1[1])))
t2Rect.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))
# Get mask by filling triangle
mask = np.zeros((r[3], r[2], 3), dtype = np.float32)
cv2.fillConvexPoly(mask, np.int32(tRect), (1.0, 1.0, 1.0), 16, 0)
# Apply warpImage to small rectangular patches
img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]
img2Rect = img2[r2[1]:r2[1] + r2[3], r2[0]:r2[0] + r2[2]]
size = (r[2], r[3])
warpImage1 = apply_affine_transform(img1Rect, t1Rect, tRect, size)
warpImage2 = apply_affine_transform(img2Rect, t2Rect, tRect, size)
# Alpha blend rectangular patches
imgRect = (1.0 - alpha) * warpImage1 + alpha * warpImage2
# Copy triangular region of the rectangular patch to the output image
img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] = img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] * ( 1 - mask ) + imgRect * mask
def generate_morph_sequence(duration,frame_rate,img1,img2,points1,points2,tri_list,size,output):
num_images = int(duration*frame_rate)
p = Popen(['ffmpeg', '-y', '-f', 'image2pipe', '-r', str(frame_rate),'-s',str(size[1])+'x'+str(size[0]), '-i', '-', '-c:v', 'libx264', '-crf', '25','-vf','scale=trunc(iw/2)*2:trunc(ih/2)*2','-pix_fmt','yuv420p', output], stdin=PIPE)
for j in range(0, num_images):
# Convert Mat to float data type
img1 = np.float32(img1)
img2 = np.float32(img2)
# Read array of corresponding points
points = []
alpha = j/(num_images-1)
# Compute weighted average point coordinates
for i in range(0, len(points1)):
x = (1 - alpha) * points1[i][0] + alpha * points2[i][0]
y = (1 - alpha) * points1[i][1] + alpha * points2[i][1]
points.append((x,y))
# Allocate space for final output
morphed_frame = np.zeros(img1.shape, dtype = img1.dtype)
for i in range(len(tri_list)):
x = int(tri_list[i][0])
y = int(tri_list[i][1])
z = int(tri_list[i][2])
t1 = [points1[x], points1[y], points1[z]]
t2 = [points2[x], points2[y], points2[z]]
t = [points[x], points[y], points[z]]
# Morph one triangle at a time.
morph_triangle(img1, img2, morphed_frame, t1, t2, t, alpha)
pt1 = (int(t[0][0]), int(t[0][1]))
pt2 = (int(t[1][0]), int(t[1][1]))
pt3 = (int(t[2][0]), int(t[2][1]))
#cv2.line(morphed_frame, pt1, pt2, (255, 255, 255), 1, 8, 0)
#cv2.line(morphed_frame, pt2, pt3, (255, 255, 255), 1, 8, 0)
#cv2.line(morphed_frame, pt3, pt1, (255, 255, 255), 1, 8, 0)
res = Image.fromarray(cv2.cvtColor(np.uint8(morphed_frame), cv2.COLOR_BGR2RGB))
res.save(p.stdin,'JPEG')
p.stdin.close()
p.wait()
def doMorphing(img1, img2, duration, frame_rate, output):
[size, img1, img2, points1, points2, list3] = generate_face_correspondences(img1, img2)
tri = make_delaunay(size[1], size[0], list3, img1, img2)
generate_morph_sequence(duration, frame_rate, img1, img2, points1, points2, tri, size, output)
# Constants
frontal_face_detector = dlib.get_frontal_face_detector()
frontal_face_predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
def face_swap(source_image, destination_image):
source_image_grayscale = cv2.cvtColor(source_image, cv2.COLOR_BGR2GRAY)
destination_image_grayscale = cv2.cvtColor(destination_image, cv2.COLOR_BGR2GRAY)
source_image_canvas = np.zeros_like(source_image_grayscale)
height, width, no_of_channels = destination_image.shape
destination_image_canvas = np.zeros((height, width, no_of_channels), np.uint8)
def index_from_array(numpyarray):
index = None
for n in numpyarray[0]:
index = n
break
return index
source_faces = frontal_face_detector(source_image_grayscale)
if len(source_faces) == 0:
return "No face detected in the source image."
for source_face in source_faces:
source_face_landmarks = frontal_face_predictor(source_image_grayscale, source_face)
source_face_landmark_points = []
for landmark_no in range(68):
x_point = source_face_landmarks.part(landmark_no).x
y_point = source_face_landmarks.part(landmark_no).y
source_face_landmark_points.append((x_point, y_point))
source_face_landmark_points_array = np.array(source_face_landmark_points, np.int32)
source_face_convexhull = cv2.convexHull(source_face_landmark_points_array)
cv2.fillConvexPoly(source_image_canvas, source_face_convexhull, 255)
source_face_image = cv2.bitwise_and(source_image, source_image, mask=source_image_canvas)
bounding_rectangle = cv2.boundingRect(source_face_convexhull)
subdivisions = cv2.Subdiv2D(bounding_rectangle)
subdivisions.insert(source_face_landmark_points)
triangles_vector = subdivisions.getTriangleList()
triangles_array = np.array(triangles_vector, dtype=np.int32)
triangle_landmark_points_list = []
for triangle in triangles_array:
index_point_1 = (triangle[0], triangle[1])
index_point_2 = (triangle[2], triangle[3])
index_point_3 = (triangle[4], triangle[5])
index_1 = np.where((source_face_landmark_points_array == index_point_1).all(axis=1))
index_1 = index_from_array(index_1)
index_2 = np.where((source_face_landmark_points_array == index_point_2).all(axis=1))
index_2 = index_from_array(index_2)
index_3 = np.where((source_face_landmark_points_array == index_point_3).all(axis=1))
index_3 = index_from_array(index_3)
triangle = [index_1, index_2, index_3]
triangle_landmark_points_list.append(triangle)
destination_faces = frontal_face_detector(destination_image_grayscale)
if len(destination_faces) == 0:
return "No face detected in the destination image."
for destination_face in destination_faces:
destination_face_landmarks = frontal_face_predictor(destination_image_grayscale, destination_face)
destination_face_landmark_points = []
for landmark_no in range(68):
x_point = destination_face_landmarks.part(landmark_no).x
y_point = destination_face_landmarks.part(landmark_no).y
destination_face_landmark_points.append((x_point, y_point))
destination_face_landmark_points_array = np.array(destination_face_landmark_points, np.int32)
destination_face_convexhull = cv2.convexHull(destination_face_landmark_points_array)
for triangle_index_points in triangle_landmark_points_list:
source_triangle_point_1 = source_face_landmark_points[triangle_index_points[0]]
source_triangle_point_2 = source_face_landmark_points[triangle_index_points[1]]
source_triangle_point_3 = source_face_landmark_points[triangle_index_points[2]]
source_triangle = np.array([source_triangle_point_1, source_triangle_point_2, source_triangle_point_3], np.int32)
source_rectangle = cv2.boundingRect(source_triangle)
(x, y, w, h) = source_rectangle
cropped_source_rectangle = source_image[y:y+h, x:x+w]
source_triangle_points = np.array([[source_triangle_point_1[0]-x, source_triangle_point_1[1]-y],
[source_triangle_point_2[0]-x, source_triangle_point_2[1]-y],
[source_triangle_point_3[0]-x, source_triangle_point_3[1]-y]], np.int32)
destination_triangle_point_1 = destination_face_landmark_points[triangle_index_points[0]]
destination_triangle_point_2 = destination_face_landmark_points[triangle_index_points[1]]
destination_triangle_point_3 = destination_face_landmark_points[triangle_index_points[2]]
destination_triangle = np.array([destination_triangle_point_1, destination_triangle_point_2, destination_triangle_point_3], np.int32)
destination_rectangle = cv2.boundingRect(destination_triangle)
(x, y, w, h) = destination_rectangle
cropped_destination_rectangle_mask = np.zeros((h, w), np.uint8)
destination_triangle_points = np.array([[destination_triangle_point_1[0]-x, destination_triangle_point_1[1]-y],
[destination_triangle_point_2[0]-x, destination_triangle_point_2[1]-y],
[destination_triangle_point_3[0]-x, destination_triangle_point_3[1]-y]], np.int32)
cv2.fillConvexPoly(cropped_destination_rectangle_mask, destination_triangle_points, 255)
source_triangle_points = np.float32(source_triangle_points)
destination_triangle_points = np.float32(destination_triangle_points)
matrix = cv2.getAffineTransform(source_triangle_points, destination_triangle_points)
warped_rectangle = cv2.warpAffine(cropped_source_rectangle, matrix, (w, h))
warped_triangle = cv2.bitwise_and(warped_rectangle, warped_rectangle, mask=cropped_destination_rectangle_mask)
new_dest_face_canvas_area = destination_image_canvas[y:y+h, x:x+w]
new_dest_face_canvas_area_gray = cv2.cvtColor(new_dest_face_canvas_area, cv2.COLOR_BGR2GRAY)
_, mask_created_triangle = cv2.threshold(new_dest_face_canvas_area_gray, 1, 255, cv2.THRESH_BINARY_INV)
warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=mask_created_triangle)
new_dest_face_canvas_area = cv2.add(new_dest_face_canvas_area, warped_triangle)
destination_image_canvas[y:y+h, x:x+w] = new_dest_face_canvas_area
final_destination_canvas = np.zeros_like(destination_image_grayscale)
final_destination_face_mask = cv2.fillConvexPoly(final_destination_canvas, destination_face_convexhull, 255)
final_destination_canvas = cv2.bitwise_not(final_destination_face_mask)
destination_face_masked = cv2.bitwise_and(destination_image, destination_image, mask=final_destination_canvas)
destination_with_face = cv2.add(destination_face_masked, destination_image_canvas)
(x, y, w, h) = cv2.boundingRect(destination_face_convexhull)
destination_face_center_point = (int((x+x+w)/2), int((y+y+h)/2))
seamless_cloned_face = cv2.seamlessClone(destination_with_face, destination_image, final_destination_face_mask, destination_face_center_point, cv2.NORMAL_CLONE)
return seamless_cloned_face
# Constants
PREDICTOR_PATH = "./shape_predictor_68_face_landmarks.dat"
SCALE_FACTOR = 1
FEATHER_AMOUNT = 11
# Initialize Dlib models
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(PREDICTOR_PATH)
# Define helper functions
def get_landmarks(im):
rects = detector(im, 1)
if len(rects) == 0:
raise Exception("No faces detected")
landmarks = [np.matrix([[p.x, p.y] for p in predictor(im, rect).parts()]) for rect in rects]
return landmarks
def draw_convex_hull(im, points, color):
points = cv2.convexHull(points)
cv2.fillConvexPoly(im, points, color=color)
def get_face_mask(im, landmarks):
mask = np.zeros(im.shape[:2], dtype=np.float64)
for landmark in landmarks:
draw_convex_hull(mask, landmark, color=1)
mask = (cv2.GaussianBlur(mask, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0).astype(np.float64)
mask = cv2.GaussianBlur(mask, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)
return mask
def process_image(image):
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
landmarks = get_landmarks(image)
mask = get_face_mask(image, landmarks)
output_image = (mask[:, :, None] * image).astype(np.uint8)
combined_image = np.hstack([image, output_image])
return combined_image
import gradio as gr
import cv2
# Define individual functions for each interface
# Face Segmentation
def gradio_face_processing(image):
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
try:
result = process_image(image) # Assume process_image is defined elsewhere
return result
except Exception as e:
return str(e)
face_segmentation_demo = gr.Interface(
fn=gradio_face_processing,
inputs=gr.Image(label="Input Image", type="numpy"),
outputs=gr.Image(label="Processed Output"),
examples=[["1.jpg"], ["2.jpg"]],
title="Face Segmentation Tool",
description="Processes the input image to segment the face."
)
# Face Swapping
def gradio_face_swap(img1, img2):
img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)
img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
result = face_swap(img1, img2) # Assume face_swap is defined elsewhere
if isinstance(result, str):
return result # Return error message
return cv2.cvtColor(result, cv2.COLOR_BGR2RGB) # Convert result to RGB for Gradio
face_swapping_demo = gr.Interface(
fn=gradio_face_swap,
inputs=[gr.Image(label="Source Image", type="numpy"), gr.Image(label="Destination Image", type="numpy")],
outputs=gr.Image(label="Result"),
examples=[["1.jpg", "2.jpg"]],
title="Face Swapping Tool",
description="Swaps the face from the source image to the destination image."
)
# Face Morphing
def morph_with_gradio(image1, image2, duration, frame_rate):
output_path = "output.mp4"
image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2RGB)
doMorphing(image1, image2, duration, frame_rate, output_path) # Assume doMorphing is defined elsewhere
return output_path
face_morphing_demo = gr.Interface(
fn=morph_with_gradio,
inputs=[
gr.Image(label="Source Image", type="numpy"),
gr.Image(label="Destination Image", type="numpy"),
gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Duration (seconds)"),
gr.Slider(minimum=1, maximum=20, value=15, step=1, label="Frame Rate (fps)")
],
outputs=gr.Video(label="Morph Video"),
examples=[["1.jpg", "2.jpg"]],
title="Face Morphing Tool",
description="Generates a morph video by transforming one face into another."
)
# Edge Detection
def edge_detection(image, threshold1, threshold2):
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
edges = cv2.Canny(gray, threshold1, threshold2)
return edges
edge_demo = gr.Interface(
fn=edge_detection,
inputs=[
gr.Image(label="Input Image"),
gr.Slider(0, 255, value=100, step=1, label="Threshold 1"),
gr.Slider(0, 255, value=200, step=1, label="Threshold 2")
],
outputs="image",
examples=[["mona.png"]],
title="Edge Detection",
description="Detects edges in the input image using Canny edge detection with adjustable thresholds."
)
# Blurring
def blur(image, kernel_size):
kernel_size = int(kernel_size) # Ensure the kernel size is an integer
if kernel_size % 2 == 0:
kernel_size += 1 # Ensure the kernel size is odd
blurred = cv2.GaussianBlur(image, (kernel_size, kernel_size), 0)
return blurred
blur_demo = gr.Interface(
fn=blur,
inputs=[
gr.Image(label="Input Image"),
gr.Slider(1, 49, value=15, step=2, label="Kernel Size (Odd Number Only)")
],
outputs="image",
examples=[["mona.png"]],
title="Image Blurring",
description="Applies Gaussian blur to the input image with adjustable kernel size."
)
# Grayscale Conversion
# Grayscale Conversion (unchanged)
def gray(image):
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
return gray_image
gray_demo = gr.Interface(
fn=gray,
inputs="image",
outputs="image",
examples=["mona.png"],
title="Grayscale Conversion",
description="Converts the input image to grayscale."
)
# Combine all interfaces into a tabbed layout
demo = gr.TabbedInterface(
[
face_segmentation_demo,
face_swapping_demo,
face_morphing_demo,
edge_demo,
blur_demo,
gray_demo
],
["Face Segmentation", "Face Swapping", "Face Morphing", "Edge Detection", "Blurring", "Grayscale Conversion"],
theme=gr.themes.Monochrome()
)
demo.launch(share=True)