Upload 4 files

app.py

@@ -0,0 +1,48 @@
+from f_segment_img import *
+from f_measurents import *
+import gradio as gr
+import dotenv
+import ast
+dotenv.load_dotenv()
+# 
+def create_sam():
+    sam_checkpoint = "sam_vit_h_4b8939.pth"
+    model_type = "vit_h"; device = "cuda"
+    sam = sam_model_registry[model_type](checkpoint=sam_checkpoint)
+    return sam
+
+def plt2arr(fig, draw=True):
+    if draw: fig.canvas.draw()
+    rgba_buf = fig.canvas.buffer_rgba()
+    (w,h) = fig.canvas.get_width_height()
+    rgba_arr = np.frombuffer(rgba_buf, dtype=np.uint8).reshape((h,w,4))
+    return rgba_arr
+
+def frame_size_width_mm(dropdown_label):
+    if dropdown_label == 'Small (xx mm)': frame_width_px = 145
+    elif dropdown_label == 'Medium (xx mm)': frame_width_px = 150
+    elif dropdown_label == 'Large (xx mm)': frame_width_px = 155
+    return frame_width_px
+
+# 
+def ipd_app(image,dropdown_label):
+    # Measure image
+    landmarks = ast.literal_eval(os.environ['landmarks'])
+    frame_processed, measurements = measure_landmarks_img(image, landmarks, plot_landmarks_on_img = True, plot_data_on_img = True)
+    # Segment Frame
+    image, img_cropped, masks_selection, objects_segmented = segment_frame_from_img(image, landmarks, create_sam())
+    # Calibrate measurements
+    frame_width_px = get_frame_width(masks_selection)
+    frame_width_mm = frame_size_width_mm(dropdown_label)
+    ipd_mm = ipd_calibration(measurements['ipd_px'], frame_width_px, frame_width_mm)
+    text_ipd = 'IPD: ' + str(round(ipd_mm,2)) + ' mm'
+    # Check
+    sam_check = plot_sam_check_segmentation_frame(image, img_cropped, objects_segmented)
+    sam_check_numpy = plt2arr(sam_check, draw = True)
+    # 
+    return text_ipd, frame_processed, str(measurements), sam_check_numpy
+
+
+dropdown = gr.Dropdown(["Small (xx mm)", "Medium (xx mm)", "Large (xx mm)"], label="Refractives Frame Size", info="For calibration")
+demo = gr.Interface(fn=ipd_app, inputs=["image",dropdown], outputs=["text", "image", "text", "image"])
+demo.launch(debug=True)

f_measurents.py

@@ -0,0 +1,490 @@
+import cv2
+import numpy as np
+import mediapipe as mp
+import matplotlib.pyplot as plt
+import math
+import imutils
+from IPython import display
+import time
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go 
+from PIL import Image, ExifTags
+
+# ---------------------------------
+# GEOMETRY TOOLS
+# ---------------------------------
+
+def distanceCalculate(p):
+    p1 = p[0]
+    p2 = p[1]
+    dis = ((p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2) ** 0.5
+    return dis
+
+def angleLinePoints(p):
+    p1 = p[0]
+    p2 = p[1]
+    # 
+    p1_x = p1[0]
+    p1_y = p1[1]
+    p2_x = p2[0]
+    p2_y = p2[1]
+    # 
+    d_x = p2_x - p1_x
+    d_y = p2_y - p1_y
+    # 
+    angle_radians = np.arctan(d_y/d_x)
+    angle_degrees = math.degrees(angle_radians)
+    return angle_degrees
+
+def area_px_within_polyline(stacked_array):
+    x = [stacked_array[0] for stacked_array in stacked_array]
+    y = [stacked_array[1] for stacked_array in stacked_array]
+    return 0.5*np.abs(np.dot(x,np.roll(y,1))-np.dot(y,np.roll(x,1)))
+
+def focal_length_calculator(measured_distance, real_width, width_in_rf_image):
+    # https://www.geeksforgeeks.org/realtime-distance-estimation-using-opencv-python/
+    focal_length = (width_in_rf_image* measured_distance)/ real_width
+    return focal_length
+
+def distance_camera_to_face_calculator(focal_length, real_face_width, face_width_in_frame):
+    distance = (real_face_width * focal_length)/face_width_in_frame
+    return distance
+
+def put_text_args(height, width, n_lines, scale):
+    font_scale = int(min(width,height)/(350/scale))
+    font_thickness = int(min(width,height)/500)
+    # 
+    line_width = int(font_thickness)
+    point_width = int(font_thickness)
+    thickness_oval = int(1.5*font_thickness)
+    # 
+    x_position_0 = int(width/20)
+    top_padding = height/20
+    text_block_height = height*0.6
+    line_heigh_increase = text_block_height/n_lines
+    y_position_v = [top_padding]
+    for i in range(1,n_lines):
+        y_position_temp = int(y_position_v[i-1] + line_heigh_increase)
+        y_position_v.append(y_position_temp)
+    # 
+    return font_scale, font_thickness, line_width, point_width, thickness_oval, x_position_0, y_position_v
+
+# ---------------------------------
+# IMAGE PROCESSING TOOLS
+# ---------------------------------
+
+def bgr_image(image):
+    """ The code takes in an image as input and splits it into three colors: 
+    blue (B), green (G), and red (R)."""
+    (B, G, R) = cv2.split(image)
+    return B, G, R
+
+def height_width_image(image):
+    """ The code will return the height and width of an image."""
+    height, width, _ = image.shape
+    return height, width
+
+def image_bgr_to_rgb(image):
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    return image
+
+def image_rgb_to_bgr(image):
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    return image
+
+def extract_exif_metadata(image_path):
+    try:
+        img = Image.open(image_path)
+        exif = { ExifTags.TAGS[k]: v for k, v in img._getexif().items() if k in ExifTags.TAGS }
+        return exif
+    except Exception as e: print(e), print("Check if EXIF data is availible for this image.")
+
+def focal_length_metadata(image_path):
+    focal_length = 0
+    focal_length_in_35mm_film = 0
+    try:
+        exif = extract_exif_metadata(image_path)
+        focal_length = exif['FocalLength']
+        focal_length_in_35mm_film = exif['FocalLengthIn35mmFilm']
+    except Exception as e: 
+        print(e), print("Check if EXIF data is availible for this image.")
+    # 
+    return focal_length, focal_length_in_35mm_film
+
+
+# ---------------------------------
+# FACE LANDMARKS
+# --------------------------------- 
+
+
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(
+        static_image_mode=True,
+        max_num_faces=1,
+        refine_landmarks=True,
+        min_detection_confidence=0.5)
+
+
+def face_mesh_points(image):
+    result = face_mesh.process(image)
+    height, width = height_width_image(image)
+    face = result.multi_face_landmarks
+    NoneType = type(None)
+    if isinstance(face, NoneType):
+        return None, image
+    mesh_points= np.array([np.multiply([p.x, p.y], [width, height]).astype(int) for p in result.multi_face_landmarks[0].landmark])
+    return result, mesh_points
+
+def find_iris_location(mesh_points,landmarks):
+    (l_cx, l_cy), l_radius = cv2.minEnclosingCircle(mesh_points[landmarks['rightEyeIris']])
+    (r_cx, r_cy), r_radius = cv2.minEnclosingCircle(mesh_points[landmarks['leftEyeIris']])
+    center_left_iris = np.array([l_cx, l_cy], dtype=np.int32)
+    center_right_iris = np.array([r_cx, r_cy], dtype=np.int32)
+    iris_position = [center_left_iris,center_right_iris]
+    iris_radius = [l_radius, r_radius]
+    # 
+    return iris_position, iris_radius
+
+# ---------------------------------
+# INTERACTIVE PLOTTTIN - FACE LANDMARKS 
+# --------------------------------- 
+
+def mesh_points_to_df(mesh_points):
+    df_mesh_points = pd.DataFrame(columns=['X_pos', 'Y_pos', 'idx'])
+    # points_x = [mesh_points[0] for stacked_array in mesh_points]
+    # points_y = [mesh_points[1] for stacked_array in mesh_points]
+    for i, point in enumerate(mesh_points):
+        point_x = point[0]
+        point_y = point[1]
+        df_mesh_points.at[i,'X_pos'] = point_x
+        df_mesh_points.at[i,'Y_pos'] = point_y
+        df_mesh_points.at[i,'idx'] = i
+    return df_mesh_points
+
+def mesh_points_interactive_plot(mesh_points):
+    df_mesh_points = mesh_points_to_df(mesh_points)
+    fig = px.scatter(df_mesh_points, y="Y_pos", x="X_pos",hover_data=['idx'])
+    fig.update_traces(marker_size=5)
+    fig['layout']['yaxis']['autorange'] = "reversed"
+    fig.write_html("output/landmarks_mesh.html")
+    return fig
+
+def mesh_points_interactive_plot_with_image(mesh_points, image_path):
+    image_temp = cv2.imread(image_path)
+    df_mesh_points = mesh_points_to_df(mesh_points)
+    height, width = height_width_image(image_temp)
+    # 
+    fig = go.Figure()
+    fig.add_layout_image(
+            x=0,
+            sizex=width,
+            y=0,
+            sizey=height,
+            xref="x",
+            yref="y",
+            opacity=1.0,
+            layer="below",
+            source=image_path
+    )
+    fig.add_scatter(x=df_mesh_points['X_pos'],y=df_mesh_points['Y_pos'],mode="markers",marker=dict(size=1, color="Red"))
+    fig.update_xaxes(showgrid=False, range=(0, width))
+    fig.update_yaxes(showgrid=False, scaleanchor='x', range=(height, 0))
+    fig.update_layout(xaxis_range=[0,width])
+    fig.write_html("output/landmarks_mesh_with_photo.html")
+    fig.write_image("output/landmarks_mesh_with_photo.png")
+    fig.write_image("output/landmarks_mesh_with_photo.svg")
+    fig.write_image("output/landmarks_mesh_with_photo.pdf")
+    return fig
+
+# ---------------------------------
+# MEASUREMENTS FACE LANDMARKS
+# --------------------------------- 
+
+def get_face_dimensions_px(mesh_points, landmarks):
+    width_face_px = distanceCalculate(mesh_points[landmarks['leftToRight']])
+    height_face_px = distanceCalculate(mesh_points[landmarks['topToBottom']])
+    return width_face_px, height_face_px
+
+def get_ipd_px(iris_position):
+    center_left_iris = iris_position[0]
+    center_right_iris = iris_position[1]
+    ipd_px = distanceCalculate([center_left_iris, center_right_iris])
+    return ipd_px
+
+def area_px_right_silhoutte_calc(mesh_points, landmarks):
+    right_silhoutte = mesh_points[landmarks['rightSilhouette']]
+    area_px_right_silhoutte = area_px_within_polyline(right_silhoutte)
+    return area_px_right_silhoutte
+
+def area_px_left_silhoutte_calc(mesh_points, landmarks):
+    left_silhoutte = mesh_points[landmarks['leftSilhouette']]
+    area_px_left_silhoutte = area_px_within_polyline(left_silhoutte)
+    return area_px_left_silhoutte
+
+def get_top_to_bottom_angle(mesh_points, landmarks):
+    top_to_bottom_angle = angleLinePoints(mesh_points[landmarks['topToBottom']])
+    return top_to_bottom_angle
+
+def get_left_to_right_angle(mesh_points, landmarks):
+    left_to_right_angle = angleLinePoints(mesh_points[landmarks['leftToRight']])
+    return left_to_right_angle
+
+def get_left_cheek_to_nose_angle(mesh_points, landmarks):
+    left_to_right_angle = angleLinePoints(mesh_points[landmarks['leftCheekToNose']])
+    return left_to_right_angle
+
+def get_nose_to_right_cheek_angle(mesh_points, landmarks):
+    left_to_right_angle = angleLinePoints(mesh_points[landmarks['noseToRightCheek']])
+    return left_to_right_angle
+
+
+# ---------------------------------
+# DRAW LANDMARKS
+# --------------------------------- 
+colour_line = (255, 0, 0)
+colour_point = (255, 0, 0)
+colour_rectangle = (255, 255, 255)
+colour_oval = (255, 255, 255)
+colour_point_iris = (0, 255, 0)
+
+def print_face_mesh_image(image, result):
+    height, width = height_width_image(image)
+    for facial_landmarks in result.multi_face_landmarks:
+        for i in range(0, 468):
+            pt1 = facial_landmarks.landmark[i]
+            x = int(pt1.x * width)
+            y = int(pt1.y * height)
+            image = cv2.circle(image, (x, y), 1, colour_point, point_width)
+    return image
+
+
+def print_iris_location(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['rightEyeIris']]], 1, colour_line, line_width)
+    image = cv2.polylines(image, [mesh_points[landmarks['leftEyeIris']]], 1, colour_line, line_width)
+    return image
+
+def print_center_iris(image, iris_position):
+    center_left_iris = iris_position[0]
+    center_right_iris = iris_position[1]
+    image = cv2.circle(image, center_left_iris, 1, colour_point_iris, 10*point_width)
+    image = cv2.circle(image, center_right_iris, 1, colour_point_iris, 10*point_width)
+    return image
+
+def print_line_left_to_right_iris(image, iris_position):
+    center_left_iris = iris_position[0]
+    center_right_iris = iris_position[1]
+    image = cv2.line(image, center_left_iris, center_right_iris, colour_line, line_width)
+    # 
+    return image
+
+def print_line_top_to_bottom(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['topToBottom']]],  1, colour_line, line_width)
+    return image
+
+def print_line_left_to_right(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['leftToRight']]],  1, colour_line, line_width)
+    return image
+
+def print_line_left_cheek_to_nose(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['leftCheekToNose']]],  1, colour_line, line_width)
+    return image
+
+def print_line_nose_to_right_cheek(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['noseToRightCheek']]],  1, colour_line, line_width)
+    return image
+
+def print_silhouette(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['silhouette']]],  1, colour_line, line_width)
+    return image
+
+def print_right_silhouette(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['rightSilhouette']]],  1, colour_line, line_width)
+    return image
+
+def print_left_silhouette(image, mesh_points, landmarks):
+    image = cv2.polylines(image, [mesh_points[landmarks['leftSilhouette']]],  1, colour_line, line_width)
+    return image
+
+def print_rectangle_card_area(image, mesh_points, landmarks):
+    _, height_face_px = get_face_dimensions_px(mesh_points, landmarks)
+    x_start = mesh_points[landmarks['outerRightEyebrowUpper']][0]
+    y_start = mesh_points[landmarks['outerRightEyebrowUpper']][1] - 0.6*height_face_px
+    x_end, y_end =  mesh_points[landmarks['outerLeftEyebrowUpper']]
+    start_point = (int(x_start), int(y_start))
+    end_point = (int(x_end), int(y_end))
+    image = cv2.rectangle(image, start_point, end_point, colour_rectangle, line_width)
+    return image
+
+def print_face_oval(image):
+    height, width = height_width_image(image)
+    image = cv2.ellipse(image, center=(int(width/2), int(height/2)), axes=(int(min(width,height)/4),int(min(width,height)/3)), angle=0, startAngle=0, endAngle=360, color=colour_oval, thickness=thickness_oval)  
+    return image
+
+# ---------------------------------
+# SCREEN-PRINTING
+# ---------------------------------
+
+# Printing text information onscreen
+colour_text =  (255, 255, 0)
+colour_text_valid = (0,255,0)
+colour_text_invalid = (255, 0, 0)
+# 
+
+def screenprint_top_to_bottom_angle(image, top_to_bottom_angle, top_to_bottom_angle_ref, top_to_bottom_angle_max_deviation):
+    if abs(top_to_bottom_angle - top_to_bottom_angle_ref) < top_to_bottom_angle_max_deviation:
+        colour_text_top_to_bottom_angle = colour_text_valid
+    else:
+        colour_text_top_to_bottom_angle = colour_text_invalid
+    image = cv2.putText(image, f'top_to_bottom_angle: {str(round(top_to_bottom_angle,3))} [degrees]', (int(x_position_0),int(y_position_v[0])), cv2.FONT_HERSHEY_PLAIN, font_scale, colour_text_top_to_bottom_angle, font_thickness)
+    return image
+
+def screenprint_top_to_bottom_angle_simple(image, top_to_bottom_angle):
+    image = cv2.putText(image, f'top_to_bottom_angle: {str(round(top_to_bottom_angle,3))} [degrees]', (int(x_position_0),int(y_position_v[0])), cv2.FONT_HERSHEY_PLAIN, font_scale, colour_text, font_thickness)
+    return image
+
+def screenprint_left_to_right_angle(image, left_to_right_angle, left_to_right_angle_ref, left_to_right_angle_max_deviation_perc):
+    if abs(left_to_right_angle - left_to_right_angle_ref) < left_to_right_angle_max_deviation_perc:
+        colour_text_left_to_right_angle = colour_text_valid
+    else:
+        colour_text_left_to_right_angle = colour_text_invalid
+    image = cv2.putText(image, f'left_to_right_angle: {str(round(left_to_right_angle,3))} [degrees]', (x_position_0,int(y_position_v[1])), cv2.FONT_HERSHEY_PLAIN, font_scale, colour_text_left_to_right_angle, font_thickness)
+    return image
+
+def screenprint_left_to_right_angle_simple(image, left_to_right_angle):
+    image = cv2.putText(image, f'left_to_right_angle: {str(round(left_to_right_angle,3))} [degrees]', (x_position_0,int(y_position_v[1])), cv2.FONT_HERSHEY_PLAIN, font_scale, colour_text, font_thickness)
+    return image
+
+def screenprint_ipd_px(image, ipd_px):
+    image = cv2.putText(image, f'ipd_px: {str(round(ipd_px,3))} [px]', (x_position_0,int(y_position_v[2])), cv2.FONT_HERSHEY_PLAIN, font_scale, colour_text, font_thickness)
+    return image
+
+def screenprint_area_right_to_left_silhoutte(image, area_right_to_left_silhoutte, area_ratio_right_to_left_ref, area_ratio_right_to_left_max_deviation_perc):
+    if abs(area_right_to_left_silhoutte - area_ratio_right_to_left_ref) < area_ratio_right_to_left_max_deviation_perc:
+        colour_text_area_right_to_left_silhoutte = colour_text_valid
+    else:
+        colour_text_area_right_to_left_silhoutte = colour_text_invalid
+    image = cv2.putText(image, f'area_right_to_left_silhoutte: {str(round(area_right_to_left_silhoutte,3))} [%]', (x_position_0,int(y_position_v[3])), cv2.FONT_HERSHEY_PLAIN, font_scale,colour_text_area_right_to_left_silhoutte, font_thickness)
+    return image
+
+def screenprint_area_right_to_left_silhoutte_simple(image, area_right_to_left_silhoutte):
+    image = cv2.putText(image, f'area_right_to_left_silhoutte: {str(round(area_right_to_left_silhoutte,3))} [%]', (x_position_0,int(y_position_v[3])), cv2.FONT_HERSHEY_PLAIN, font_scale,colour_text, font_thickness)
+    return image
+
+def screenprint_nose_to_cheek(image,left_cheek_to_nose_angle,nose_to_right_cheek_angle):
+    image = cv2.putText(image, f'Nose-Cheek Angles: {str(round(left_cheek_to_nose_angle,3)), str(round(nose_to_right_cheek_angle,3))} [degrees]', (x_position_0,int(y_position_v[4])), cv2.FONT_HERSHEY_PLAIN, font_scale,colour_text, font_thickness)
+    return image
+
+
+# ---------------------------------
+# GET ALL MEASUREMENT DATA FUNCTION
+# ---------------------------------
+
+def get_measurements_from_landmarks(mesh_points,landmarks):
+    # 
+    iris_position, iris_radius = find_iris_location(mesh_points,landmarks)
+    ipd_px = get_ipd_px(iris_position)
+    width_face_px, height_face_px = get_face_dimensions_px(mesh_points, landmarks)
+    top_to_bottom_angle = get_top_to_bottom_angle(mesh_points, landmarks)
+    left_to_right_angle = get_left_to_right_angle(mesh_points, landmarks)
+    left_cheek_to_nose_angle = get_left_cheek_to_nose_angle(mesh_points, landmarks)
+    nose_to_right_cheek_angle = get_nose_to_right_cheek_angle(mesh_points, landmarks)
+    area_px_left_silhoutte = area_px_left_silhoutte_calc(mesh_points, landmarks)
+    area_px_right_silhoutte = area_px_right_silhoutte_calc(mesh_points, landmarks)
+    area_right_to_left_silhoutte = (1 - (area_px_right_silhoutte/area_px_left_silhoutte))*100
+    # 
+    # Create dictionary to return measurements
+    measurements = {'iris_position': iris_position, 
+                    'iris_radius': iris_radius, 
+                    'ipd_px': ipd_px,
+                    'width_face_px': width_face_px,
+                    'height_face_px': height_face_px,
+                    'top_to_bottom_angle': top_to_bottom_angle,
+                    'left_to_right_angle': left_to_right_angle,
+                    'left_cheek_to_nose_angle': left_cheek_to_nose_angle,
+                    'nose_to_right_cheek_angle': nose_to_right_cheek_angle,
+                    'area_px_left_silhoutte': area_px_left_silhoutte,
+                    'area_px_right_silhoutte': area_px_right_silhoutte,
+                    'area_right_to_left_silhoutte': area_right_to_left_silhoutte
+                    }
+    # 
+    return measurements
+
+
+# ---------------------------------
+# LINE PROPERTIES
+# ---------------------------------
+
+def define_plotting_properties(image):
+    n_lines = 5
+    scale = 1
+     # Properties of lines
+    global font_scale, font_thickness, line_width, point_width, thickness_oval, x_position_0, y_position_v
+    height, width = height_width_image(image)
+    font_scale, font_thickness, line_width, point_width, thickness_oval, x_position_0, y_position_v = put_text_args(height, width, n_lines, scale)
+    if line_width == 0: line_width = 1
+
+
+# ---------------------------------
+# PRINT LANDMARKS, MEASUREMENTS AND CHECKS
+# ---------------------------------
+
+def print_landmarks_on_img(image, result, mesh_points, landmarks, iris_position):
+    image = print_face_mesh_image(image, result)
+    image = print_iris_location(image, mesh_points, landmarks)
+    image = print_center_iris(image, iris_position)
+    image = print_line_left_to_right_iris(image, iris_position)
+    image = print_line_top_to_bottom(image, mesh_points, landmarks)
+    image = print_line_left_to_right(image, mesh_points, landmarks)
+    image = print_line_left_cheek_to_nose(image, mesh_points, landmarks)
+    image = print_line_nose_to_right_cheek(image, mesh_points, landmarks)
+    image = print_silhouette(image, mesh_points, landmarks)
+    image = print_rectangle_card_area(image, mesh_points, landmarks)
+    image = print_right_silhouette(image, mesh_points, landmarks)
+    image = print_left_silhouette(image, mesh_points, landmarks)
+    image = print_face_oval(image)
+    return image
+
+def screenprint_data_and_criteria_on_img(image, measurements, criteria):
+    image = screenprint_top_to_bottom_angle(image, measurements['top_to_bottom_angle'], criteria['top_to_bottom_angle_ref'],  criteria['top_to_bottom_angle_max_deviation'])
+    image = screenprint_left_to_right_angle(image, measurements['left_to_right_angle'], criteria['left_to_right_angle_ref'], criteria['left_to_right_angle_max_deviation_perc'])
+    image = screenprint_area_right_to_left_silhoutte(image, measurements['area_right_to_left_silhoutte'], criteria['area_ratio_right_to_left_ref'], criteria['area_ratio_right_to_left_max_deviation_perc'])
+    image = screenprint_ipd_px(image, measurements['ipd_px'])
+    image = screenprint_nose_to_cheek(image,measurements['left_cheek_to_nose_angle'],measurements['nose_to_right_cheek_angle'])
+    return image
+
+def screenprint_data_on_img(image, measurements):
+    image = screenprint_top_to_bottom_angle_simple(image, measurements['top_to_bottom_angle'])
+    image = screenprint_left_to_right_angle_simple(image, measurements['left_to_right_angle'])
+    image = screenprint_area_right_to_left_silhoutte_simple(image, measurements['area_right_to_left_silhoutte'])
+    image = screenprint_ipd_px(image, measurements['ipd_px'])
+    image = screenprint_nose_to_cheek(image,measurements['left_cheek_to_nose_angle'],measurements['nose_to_right_cheek_angle'])
+    return image
+
+# ---------------------------------
+# MAIN FUNCTION
+# ---------------------------------
+
+def measure_landmarks_img(image, landmarks, plot_landmarks_on_img = True, plot_data_on_img = True):
+    # 
+    try: image = image_bgr_to_rgb(image)
+    except Exception as e: print(e)
+    # Create global parameters for plotting properties (lines, points / width, colour, etc.)
+    define_plotting_properties(image)
+    # Getting face lanmarks
+    result, mesh_points = face_mesh_points(image)
+    # Measurements
+    measurements = get_measurements_from_landmarks(mesh_points,landmarks)
+    # Printing objects
+    if plot_landmarks_on_img == True: image = print_landmarks_on_img(image, result, mesh_points, landmarks, measurements['iris_position'])
+    # Screenprinting data and checks
+    if plot_data_on_img == True: image = screenprint_data_on_img(image, measurements)
+    # Convert to RGB
+    image = image_bgr_to_rgb(image)
+    # Return image
+    return image, measurements
+
+
+# ---------------------------------
+# OTHER FUNCTIONS
+# ---------------------------------

f_segment_img.py

@@ -0,0 +1,181 @@
+from f_measurents import *
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import mediapipe as mp
+from mediapipe.tasks.python import vision
+import sys
+import os
+from segment_anything import sam_model_registry, SamAutomaticMaskGenerator, SamPredictor
+#
+sys.path.insert(
+    1, '/Users/danielfiuzadosil/Documents/GitHub_Repo/Bryant_Medical/eCommerce/App_IPD [Master]/ipd_app/src/modules')
+#
+
+
+def remove_background_img(img):
+    THRESHOLD = 0.12
+    # initialize mediapipe
+    mp_selfie_segmentation = mp.solutions.selfie_segmentation
+    selfie_segmentation = mp_selfie_segmentation.SelfieSegmentation(
+        model_selection=1)
+    # get the result
+    results = selfie_segmentation.process(img)
+    # extract segmented mask
+    mask = np.stack((results.segmentation_mask,) * 3, axis=-1) > THRESHOLD
+    mask_binary = mask.astype(int)*255
+    img_masked = img.copy()
+    img_masked[mask_binary == 0] = 255
+    #
+    return img_masked
+
+
+def remove_background_img_v2(image_path, MODEL_PATH='/Users/danielfiuzadosil/Documents/GitHub_Repo/Bryant_Medical/eCommerce/App_IPD [Master]/ipd_app/data/external/mediapipe_models/deeplabv3.tflite'):
+    #
+    THRESHOLD = 0.12
+    #
+    BG_COLOR = (0, 0, 0)  # black
+    MASK_COLOR = (255, 255, 255)  # white
+    #
+    BaseOptions = mp.tasks.BaseOptions
+    OutputType = vision.ImageSegmenterOptions.OutputType
+    # Create the options that will be used for ImageSegmenter
+    base_options = BaseOptions(model_asset_path=MODEL_PATH)
+    options = vision.ImageSegmenterOptions(
+        base_options=base_options, output_type=OutputType.CATEGORY_MASK)
+    # Create the MediaPipe image file that will be segmented
+    image = mp.Image.create_from_file(image_path)
+    with vision.ImageSegmenter.create_from_options(options) as segmenter:
+        # Retrieve the masks for the segmented image
+        category_masks = segmenter.segment(image)
+    # Generate solid color images for showing the output segmentation mask.
+    image_data = image.numpy_view()
+    fg_image = np.zeros(image_data.shape, dtype=np.uint8)
+    fg_image[:] = MASK_COLOR
+    bg_image = np.zeros(image_data.shape, dtype=np.uint8)
+    bg_image[:] = BG_COLOR
+    #
+    condition = np.stack(
+        (category_masks[0].numpy_view(),) * 3, axis=-1) > THRESHOLD
+    mask_binary = np.where(condition, fg_image, bg_image)
+    #
+    img_masked = image_data.copy()
+    img_masked[mask_binary == 0] = 255
+    #
+    return img_masked
+
+
+def segment_frame_from_img(image, landmarks, sam):
+    # Read image
+    image_0 = image.copy()
+    # Generate facial landmarks
+    mp_face_mesh = mp.solutions.face_mesh
+    face_mesh = mp_face_mesh.FaceMesh(
+        static_image_mode=True,
+        max_num_faces=1,
+        refine_landmarks=True,
+        min_detection_confidence=0.5)
+    # Calculate facial landmaks and other data
+    result, mesh_points = face_mesh_points(image)
+    df_mesh_points = mesh_points_to_df(mesh_points)
+    width_face_px, height_face_px = get_face_dimensions_px(
+        mesh_points, landmarks)
+    # Calculate rectangle where the frame will likely be (based on a reference landmarks)
+    squareBoxEyes = mesh_points[landmarks['squareBoxEyes']]
+    #
+    width_rectangle = squareBoxEyes[2][0] - squareBoxEyes[0][0]
+    tolerance_x = 0.2*width_rectangle
+    height_rectangle = squareBoxEyes[2][1] - squareBoxEyes[0][1]
+    tolerance_y = height_rectangle*0.2
+    #
+    x_start, y_start = [squareBoxEyes[0][0] -
+                        tolerance_x, squareBoxEyes[0][1] - tolerance_y]
+    x_end, y_end = [squareBoxEyes[2][0] + tolerance_x, squareBoxEyes[2][1]]
+    # Cropped image to region where the frames will be located
+    img_cropped = image_0[int(y_start):int(y_end), int(x_start):int(x_end)]
+    # Use Meta's Segment Anything Model (SAM) to segment the frame
+    mask_generator = SamAutomaticMaskGenerator(sam)
+    masks = mask_generator.generate(img_cropped)
+    # Select the right object by defining the expected range of area occupied by the frame
+    height, width, _ = img_cropped.shape
+    area_photo = height*width
+    area_frame_min = 0.2
+    area_frame_max = 0.6
+    # Iterate through the different masks and store the ones that fulfill the criteria
+    masks_selection = []
+    objects_segmented = []
+    for i in range(len(masks)):
+        mask = masks[i]
+        area_object = mask['area']
+        if area_photo*area_frame_max > area_object > area_photo*area_frame_min:
+            masks_selection.append(mask)
+            #
+            mask_binary_temp = mask['segmentation'].astype(int)*255
+            object_segmented = img_cropped.copy()
+            object_segmented[mask_binary_temp == 0] = 255
+            objects_segmented.append(object_segmented)
+
+    return image, img_cropped, masks_selection, objects_segmented
+
+
+def plot_sam_check_segmentation_frame(image, img_cropped, objects_segmented):
+    #
+    ax1 = plt.subplot(311)
+    ax1.imshow(image)
+    ax1.axis("Off")
+    ax2 = plt.subplot(312)
+    ax2.imshow(img_cropped)
+    ax2.axis("Off")
+    ax3 = plt.subplot(313)
+    ax3.imshow(objects_segmented[0])
+    fig = plt.gcf()
+    return fig
+    
+
+def plot_sam_check_segmentation_frame_and_save(image, img_cropped, objects_segmented, output_folder, filepath):
+    #
+    ax1 = plt.subplot(311)
+    ax1.imshow(image)
+    ax1.axis("Off")
+    ax2 = plt.subplot(312)
+    ax2.imshow(img_cropped)
+    ax2.axis("Off")
+    ax3 = plt.subplot(313)
+    ax3.imshow(objects_segmented[0])
+    #
+    plt.savefig(output_folder + os.path.basename(filepath),
+                transparent=True, bbox_inches='tight')
+    # plt.show()
+
+def plot_segmented_object_with_bb(objects_segmented, masks_selection, image):
+    image = cv2.rectangle(objects_segmented[0], masks_selection[0]['bbox'], [255,0,0], 4)
+    plt.imshow(image)
+    ax = plt.gca()
+    # 
+    img_height = image.shape[0]
+    img_width = image.shape[1]
+    # 
+    major_ticks_height = np.arange(0, img_height, 100)
+    minor_ticks_height = np.arange(0, img_height, 10)
+    major_ticks_width = np.arange(0, img_width, 100)
+    minor_ticks_width = np.arange(0, img_width, 10)
+    # 
+    ax.set_yticks(major_ticks_height)
+    ax.set_yticks(minor_ticks_height, minor=True)
+    ax.set_xticks(major_ticks_width)
+    ax.set_xticks(minor_ticks_width, minor=True)
+    # 
+    ax.grid(which='both')
+    ax.grid(which='minor', alpha=0.2)
+    ax.grid(which='major', alpha=0.5)
+    # 
+    # plt.show()
+
+def get_frame_width(masks_selection):
+    frame_width = masks_selection[0]['bbox'][2]
+    return frame_width
+
+def ipd_calibration(ipd_px, frame_width_px, frame_width_mm):
+    calibration_factor = frame_width_px/frame_width_mm
+    ipd_mm = ipd_px/calibration_factor
+    return ipd_mm

ipd_gradio.ipynb

@@ -0,0 +1,183 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO: Created TensorFlow Lite XNNPACK delegate for CPU.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from f_segment_img import *\n",
+    "from f_measurents import *\n",
+    "import gradio as gr\n",
+    "import dotenv\n",
+    "import ast\n",
+    "dotenv.load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_sam():\n",
+    "    sam_checkpoint = \"sam_vit_h_4b8939.pth\"\n",
+    "    model_type = \"vit_h\"; device = \"cuda\"\n",
+    "    sam = sam_model_registry[model_type](checkpoint=sam_checkpoint)\n",
+    "    return sam\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def plt2arr(fig, draw=True):\n",
+    "    if draw: fig.canvas.draw()\n",
+    "    rgba_buf = fig.canvas.buffer_rgba()\n",
+    "    (w,h) = fig.canvas.get_width_height()\n",
+    "    rgba_arr = np.frombuffer(rgba_buf, dtype=np.uint8).reshape((h,w,4))\n",
+    "    return rgba_arr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def frame_size_width_mm(dropdown_label):\n",
+    "    if dropdown_label == 'Small (142 mm)': frame_width_px = 142\n",
+    "    elif dropdown_label == 'Medium (xx mm)': frame_width_px = 150\n",
+    "    elif dropdown_label == 'Large (xx mm)': frame_width_px = 155\n",
+    "    return frame_width_px"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def ipd_app(image,dropdown_label):\n",
+    "    # \n",
+    "    landmarks = ast.literal_eval(os.environ['landmarks'])\n",
+    "    frame_processed, measurements = measure_landmarks_img(image, landmarks, plot_landmarks_on_img = True, plot_data_on_img = True)\n",
+    "    # \n",
+    "    image, img_cropped, masks_selection, objects_segmented = segment_frame_from_img(image, landmarks, create_sam())\n",
+    "    # \n",
+    "    frame_width_px = get_frame_width(masks_selection)\n",
+    "    frame_width_mm = frame_size_width_mm(dropdown_label)\n",
+    "    ipd_mm = ipd_calibration(measurements['ipd_px'], frame_width_px, frame_width_mm)\n",
+    "    text_ipd = 'IPD: ' + str(round(ipd_mm,2)) + ' mm'\n",
+    "    # \n",
+    "    sam_check = plot_sam_check_segmentation_frame(image, img_cropped, objects_segmented)\n",
+    "    sam_check_numpy = plt2arr(sam_check, draw = True)\n",
+    "    # \n",
+    "    return text_ipd, frame_processed, str(measurements), sam_check_numpy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_test = '/Users/danielfiuzadosil/Documents/GitHub_Repo/Bryant_Medical/eCommerce/App_IPD [Master]/ipd_app/data/raw/segmentation/sample_w_frames.jpeg'\n",
+    "image = cv2.cvtColor(cv2.imread(image_test),cv2.COLOR_BGR2RGB)\n",
+    "dropdown_label = \"Small (xx mm)\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/homebrew/lib/python3.9/site-packages/gradio/deprecation.py:43: UserWarning: You have unused kwarg parameters in Dropdown, please remove them: {'info': 'For calibration'}\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "IMPORTANT: You are using gradio version 3.7, however version 3.14.0 is available, please upgrade.\n",
+      "--------\n",
+      "Running on local URL:  http://127.0.0.1:7860\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7860/\" width=\"900\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dropdown = gr.Dropdown([\"Small (142 mm)\", \"Medium (xx mm)\", \"Large (xx mm)\"], label=\"Refractives Frame Size\", info=\"For calibration\")\n",
+    "demo = gr.Interface(fn=ipd_app, inputs=[\"image\",dropdown], outputs=[\"text\", \"image\", \"text\", \"image\"])\n",
+    "demo.launch(debug=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.15"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}