initial commit

.env

@@ -0,0 +1,2 @@
+BASE_URL = "http://127.0.0.1:7860"
+OPENAI_API_KEY = "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"

.gitignore

@@ -0,0 +1,5 @@
+models
+__pycache__
+Acne_Classifyer_N_Resnet.h5
+haarcascade_eye.xml
+shape_predictor_68_face_landmarks.dat

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10.6
+WORKDIR  /code
+
+ADD requirements.txt  /code/requirements.txt
+ADD app.py index.py model.py get_patches.py /code/
+
+RUN apt-get update && apt-get upgrade -y
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y build-essential cmake pkg-config \
+    && apt-get install -y libx11-dev libatlas-base-dev \
+    && apt-get install -y libgtk-3-dev libboost-python-dev
+
+RUN pip install --timeout=100000 -r requirements.txt
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,8 +1,8 @@
 ---
 title: SkinZen
-emoji: 📉
-colorFrom: yellow
-colorTo: yellow
+emoji: 👁
+colorFrom: blue
+colorTo: red
 sdk: docker
 pinned: false
 license: mit

app.py

@@ -0,0 +1,75 @@
+import gradio as gr
+from PIL import Image
+from io import BytesIO
+import PIL 
+import numpy as np
+import os
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+def sentence_builder(age, sex, skin_type, allergies, diet, file):
+    import index
+
+    print(age, sex, skin_type, allergies, diet)
+
+    response = index.predict(file)
+    predictions = response['prediction']
+    prediction = np.array(predictions)
+
+    data = response
+    data["prediction"] = prediction
+
+    labels = ["Low", "Moderate", "Severe"]
+    show_prediction = np.zeros((4, 3))
+
+
+    for in_, pred in enumerate(prediction):
+        show_prediction[in_] = pred
+
+    output1 = {labels[i]: float(show_prediction[0][i]) for i in range(3)}
+    output2 = {labels[i]: float(show_prediction[1][i]) for i in range(3)}
+    output3 = {labels[i]: float(show_prediction[2][i]) for i in range(3)}
+    output4 = {labels[i]: float(show_prediction[3][i]) for i in range(3)}
+
+    data['age'] = age
+    data['gender'] = sex
+    data['skin_type'] = skin_type
+    data['allergies'] = allergies
+    data['diet'] = diet
+
+
+    try:
+        response = index.recommendation(data)
+        data = response.json()
+        content = data['choices'][0]['message']['content']
+        return content, output1, output2, output3, output4
+    except: 
+        return "No recommendation found", output1, output2, output3, output4
+
+with gr.Blocks() as demo:
+    gr.Markdown("Flip text or image files using this demo.")
+    with gr.Row():
+        with gr.Column():
+            age = gr.Number(value=20, label="Age")
+            sex = gr.Radio(["Male", "Female", "Other"], label="Gender", info="Your Gender")
+            skin_type = gr.CheckboxGroup(["Oily", "Dry", "Normal"], label="Skin", info="Skin Type")
+            allergy = gr.Dropdown(
+                ["benzoyl peroxide", "salicylic acid", "Sun-exposure", "Itching", "Swelling", "Redness"],
+                multiselect=True, label="Allergies", 
+                info="Tell us your allergies and symptoms"
+            )
+            diet = gr.CheckboxGroup(["Veg", "Non-Veg",], label="Diet", info="Select your diet preference")
+            img = gr.Image(source="upload", type="pil", label="Face Image (with open eye)")
+            submit = gr.Button("Submit")
+            
+        with gr.Tab("Model:Severity Prediction"):
+            chin = gr.Label(num_top_classes=3, label="Chin|Acne Level")
+            fh = gr.Label(num_top_classes=3, label="Fore Head|Acne Level")
+            lc = gr.Label(num_top_classes=3, label="Left Cheek|Acne Level")
+            rc = gr.Label(num_top_classes=3, label="Right Cheek|Acne Level")
+        with gr.Tab("Recommendation:Treatment Plan"):
+            html_output = gr.HTML('Recommendation will be shown here')
+
+    submit.click(sentence_builder, inputs=[age, sex, skin_type, allergy, diet, img], outputs=[html_output, rc, lc, chin, fh])
+
+demo.launch()

get_patches.py

@@ -0,0 +1,334 @@
+import numpy as np
+import cv2
+from skimage import io
+import matplotlib.pyplot as plt
+from os import listdir
+from os.path import join, isfile, splitext
+from scipy import misc
+import sys
+import dlib
+import os
+from PIL import Image
+import urllib.request
+import imageio
+
+width_ratio = 1.5
+top_ratio = 1.5
+gap_ratio = 0.1
+down_ratio = 4.5
+chin_width_ratio = 2.8
+forehead_ratio = 0.3
+verb = False
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+PREDICTOR_PATH = os.path.join(BASE_DIR, "shape_predictor_68_face_landmarks.dat")
+eye_cascade = cv2.CascadeClassifier(os.path.join(BASE_DIR, "haarcascade_eye.xml"))
+
+assert not eye_cascade.empty()
+
+SCALE_FACTOR = 1 
+FEATHER_AMOUNT = 11
+
+FACE_POINTS = list(range(17, 68))
+MOUTH_POINTS = list(range(48, 61))
+RIGHT_BROW_POINTS = list(range(17, 22))
+LEFT_BROW_POINTS = list(range(22, 27))
+RIGHT_EYE_POINTS = list(range(36, 42))
+LEFT_EYE_POINTS = list(range(42, 48))
+NOSE_POINTS = list(range(27, 35))
+JAW_POINTS = list(range(0, 17))
+
+OVERLAY_POINTS = [
+    LEFT_EYE_POINTS + RIGHT_EYE_POINTS + LEFT_BROW_POINTS + RIGHT_BROW_POINTS,
+    NOSE_POINTS + MOUTH_POINTS,
+]
+
+detector = dlib.get_frontal_face_detector()
+predictor = dlib.shape_predictor(PREDICTOR_PATH)
+
+class TooManyFaces(Exception):
+    pass
+
+class NoFaces(Exception):
+    pass
+
+def get_landmarks(im):
+    rects = detector(im, 1)
+    
+    if len(rects) > 1:
+        raise TooManyFaces
+    if len(rects) == 0:
+        raise NoFaces
+
+    return np.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()])
+
+def read_imgURL(URL):
+    with urllib.request.urlopen(URL) as url:
+        with open('temp.jpg', 'wb') as f:
+            f.write(url.read())
+
+    img = Image.open('temp.jpg')
+    img = np.array(img)
+    return img
+
+def draw_convex_hull(im, points, color):
+    points = cv2.convexHull(points)
+    cv2.fillConvexPoly(im, points, color=color)
+
+def get_face_mask(im, landmarks):
+    im = np.zeros(im.shape[:2], dtype=np.float64)
+
+    for group in OVERLAY_POINTS:
+        draw_convex_hull(im,
+                         landmarks[group],
+                         color=1)
+
+    im = np.array([im, im, im]).transpose((1, 2, 0))
+
+    im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0
+    im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)
+
+    return im
+    
+
+def read_im_and_landmarks(fname):
+    im = np.array(fname)
+    im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR,
+                         im.shape[0] * SCALE_FACTOR))
+    s = get_landmarks(im)
+    return im, s
+
+def warp_im(im, M, dshape):
+    output_im = np.zeros(dshape, dtype=im.dtype)
+    cv2.warpAffine(im,
+                   M[:2],
+                   (dshape[1], dshape[0]),
+                   dst=output_im,
+                   borderMode=cv2.BORDER_TRANSPARENT,
+                   flags=cv2.WARP_INVERSE_MAP)
+    return output_im
+
+def infer_chin_region(eye, width_ratio, down_ratio, left_or_right):
+    region1 = [0] * 4
+    if left_or_right == 'right': #assuming it is the absolute right chin
+        region1[0] = int(max(0, int(eye[0] - 0.5 * eye[2]))) #chin region should go lefwards
+        region1[2] = int(0.5 * eye[2])
+    else: # assuming it is the absolute left chin
+        region1[0] = int(eye[0] + eye[2]) # chin region should go rightwards
+        region1[2] = int(0.5 * eye[2])
+    region1[1] = int(eye[1] + eye[3])
+    region1[3] = int(1.5 * eye[3])
+
+    return region1
+               
+def detect_face_direction(gray, face, eye, down_ratio, chin_width_ratio):  
+    region1 = [0] * 4 # assuming this is the left eye, forhead should go rightward
+    region2 = [0] * 4 # assuming this is the right eye, forhead should go leftward
+    print(eye[0])
+    region1 = infer_chin_region(eye[0], chin_width_ratio, down_ratio, 'left') #region1 is from eye to right
+    region2 = infer_chin_region(eye[0], chin_width_ratio, down_ratio, 'right') # region2 is from eye to left
+
+    std1 = np.std(gray[region1[1]:(region1[1]+region1[3]), region1[0]:(region1[0]+region1[2])])
+    std2 = np.std(gray[region2[1]:(region2[1]+region2[3]), region2[0]:(region2[0]+region2[2])])
+    face_direction = ""
+
+    if std1 > std2:  #eye right has higher variance than eye left
+        face_direction = "right"
+    else:
+        face_direction = "left"
+    return face_direction
+
+def extract_cheek_region(face_x_min, face_x_max, face_y_max, eye_landmarks, left_or_right):
+    if left_or_right == "Left":
+        cheek_region_min_x = eye_landmarks[0,0]
+        cheek_region_max_x = int(face_x_max - 0.05 * (face_x_max - min(eye_landmarks[:,0])))
+    else:
+        cheek_region_max_x = max(eye_landmarks[:,0])[0,0]
+        #print (max(eye_landmarks[:,0])[0,0])
+        #cheek_region_max_x = max(eye_landmarks[:, 0])
+        cheek_region_min_x = int(face_x_min + 0.1 * (cheek_region_max_x - face_x_min))
+    cheek_region_min_y = int(max(eye_landmarks[:,1]) + 0.2 * (max(eye_landmarks[:,1])  - min(eye_landmarks[:,1])))
+    cheek_region_max_y = int(face_y_max - 0.1 * (face_y_max - max(eye_landmarks[:,1])))
+    return [cheek_region_min_x, cheek_region_min_y, cheek_region_max_x, cheek_region_max_y]
+
+def extract_patches(imagefile, dimension_dict, face_loc_dict, image_dim, croppedFaces_Dir):
+    
+    imageName = "temp"
+    
+    img, landmarks = read_im_and_landmarks(imagefile)
+    face_detected = True
+     
+    img_height, img_width = img.shape[0:2]
+    image_dim = [img_height, img_width]
+    min_dim = min(img_height, img_width)
+    min_face_size = min(min_dim * 0.2, min_dim * 0.2)
+    min_eye = min_face_size * 0.2
+    min_eye_area = min_eye ** 2
+    
+    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+    
+    if face_detected:
+        mask = get_face_mask(img, landmarks)
+        face_x_min = int(max(0, np.asarray(min(landmarks[:,0])).flatten()[0]))
+        face_x_max = int(min(img_width, np.asarray(max(landmarks[:,0])).flatten()[0]))
+        face_y_min = int(max(0, np.asarray(min(landmarks[:,1])).flatten()[0]))
+        face_y_max = int(min(img_height, np.asarray(max(landmarks[:,1])).flatten()[0]))
+        face_loc_dict['face_loc'] = [face_x_min, face_x_max, face_y_min, face_y_max]
+        face_height = face_y_max - face_y_min
+        forehead_height = int(face_height * forehead_ratio)
+        new_face_y_min = max(0, face_y_min - forehead_height)
+        right_brow_landmarks = landmarks[RIGHT_BROW_POINTS,:]
+        left_brow_landmarks = landmarks[LEFT_BROW_POINTS,:]
+        right_eye_landmarks = landmarks[RIGHT_EYE_POINTS,:]
+        left_eye_landmarks = landmarks[LEFT_EYE_POINTS,:]
+        mouse_landmarks = landmarks[MOUTH_POINTS,:]
+        ########################
+        # Get the forehead patch
+        ########################
+        [right_brow_min_x, left_brow_max_x] = \
+            [max(0, np.min(np.array(right_brow_landmarks[:,0]))), min(img_width, np.max(np.array(left_brow_landmarks[:,0])))]
+        brow_min_y = min(np.min(np.array(right_brow_landmarks[:,1])),np.min(np.array(left_brow_landmarks[:,1])))
+        forehead_x_min = right_brow_min_x
+        forehead_x_max = left_brow_max_x
+        forehead_y_min = max(0, brow_min_y - forehead_height)
+        forehead_y_max = min(brow_min_y, forehead_y_min + forehead_height)
+        forehead_region = img[forehead_y_min:forehead_y_max, forehead_x_min:forehead_x_max, :]
+        #print ('forehead dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (forehead_x_min,  forehead_x_max,  forehead_y_min,  forehead_y_max))
+        key_name = 'landmark_fh'
+        dimension_dict[key_name] = [forehead_x_min, forehead_x_max, forehead_y_min, forehead_y_max]
+        forehead_file_name = join(croppedFaces_Dir, key_name +".jpg")
+        #forehead_region = cv2.cvtColor(forehead_region, cv2.COLOR_BGR2RGB)
+        imageio.imwrite(forehead_file_name, forehead_region)
+        
+        chin_x_min = np.max(np.array(right_eye_landmarks[:,0]))
+        chin_x_max = np.min(np.array(left_eye_landmarks[:,0]))
+        chin_y_min = np.max(np.array(mouse_landmarks[:,1]))
+        chin_y_max = face_y_max
+        chin_region = img[chin_y_min:chin_y_max, chin_x_min:chin_x_max, :]
+        #print ('chin dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (chin_x_min, chin_x_max, chin_y_min, chin_y_max))
+        key_name = 'landmark_chin'
+        dimension_dict[key_name] = [chin_x_min, chin_x_max, chin_y_min, chin_y_max]
+        chin_file_name = join(croppedFaces_Dir, key_name +".jpg")
+        #chin_region = cv2.cvtColor(chin_region, cv2.COLOR_BGR2RGB)
+        imageio.imwrite(chin_file_name, chin_region)
+    
+        ##########################
+        # Get the cheeks patch
+        ##########################
+        # Decide whether it is a side view or not
+        left_eye_width = np.max(np.array(left_eye_landmarks[:,0])) - np.min(np.array(left_eye_landmarks[:,0]))
+        right_eye_width = np.max(np.array(right_eye_landmarks[:,0])) - np.min(np.array(right_eye_landmarks[:,0]))
+        right_face = True
+        left_face = True
+        if float(right_eye_width) / float(left_eye_width) >= 1.15: # right eye is bigger than left eye, showing the right face
+            left_face = False
+        elif float(left_eye_width) / float(right_eye_width) >= 1.15: # left eye is bigger than right eye, showing the left face
+            right_face = False
+        
+        if right_face:
+            right_cheek_region = extract_cheek_region(face_x_min, face_x_max, face_y_max, right_eye_landmarks, "Right")
+            cheek_region = img[right_cheek_region[1]:right_cheek_region[3], right_cheek_region[0]:right_cheek_region[2], :]
+            #print ('right cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (right_cheek_region[0], right_cheek_region[2], right_cheek_region[1], right_cheek_region[3]))
+            key_name = 'landmark_rc'
+            dimension_dict[key_name] = [right_cheek_region[0], right_cheek_region[2], right_cheek_region[1], right_cheek_region[3]]
+            cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
+            #cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB)
+            imageio.imwrite(cheek_file_name, cheek_region)
+        if left_face:
+            left_cheek_region = extract_cheek_region(face_x_min, face_x_max, face_y_max, left_eye_landmarks, "Left")
+            cheek_region = img[left_cheek_region[1]:left_cheek_region[3], left_cheek_region[0]:left_cheek_region[2], :]
+            #print ('left cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (left_cheek_region[0], left_cheek_region[2], left_cheek_region[1], left_cheek_region[3]))
+            key_name = 'landmark_lc'
+            dimension_dict[key_name] = [left_cheek_region[0], left_cheek_region[2], left_cheek_region[1], left_cheek_region[3]]
+            cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
+            #cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB)
+            imageio.imwrite(cheek_file_name, cheek_region)
+    
+                
+    if not face_detected:
+        print("Face not detected by landmarks model...")
+        # Use the OneEye model to detect one eye, and infer the face region based on the eye location
+        eye_detected = False
+        roi_gray = gray
+        roi_color = img
+        roi_color = cv2.cvtColor(roi_color, cv2.COLOR_BGR2RGB)
+        eyes = eye_cascade.detectMultiScale(roi_gray, 1.1, 5)
+        max_area = 0
+        eye_count = 0
+        max_index = 0
+        
+        for (ex,ey,ew,eh) in eyes: # there might be multiple eyes detected. Choose the biggest one
+            if ew*eh >= max_area and ex >= img_width * 0.1 and ex <= img_width * 0.9:
+                max_area = ew*eh
+                max_index = eye_count
+            eye_count += 1
+        if max_area >= min_eye_area:
+            eye_detected = True
+            (ex, ey, ew, eh) = eyes[max_index]
+            if float(ew) / float(img_width) > 0.15 or float(eh) / float(img_height) > 0.15: # detected eye too large
+                # resize the detected eye
+                center_x = ex + ew/2
+                center_y = ey + eh/2
+                resized_w = min(img_width * 0.15, img_height * 0.15)
+                ex = int(center_x - resized_w/2)
+                ey = int(center_y - resized_w/2)
+                ew = int(resized_w)
+                eh = int(resized_w)
+                eyes1 = np.array([ex, ey, resized_w, resized_w]).reshape((1,4))
+            else:
+                eyes1 = np.array(eyes[max_index]).reshape((1,4))
+            face1 = np.array(())
+            face_direction = detect_face_direction(gray, face1, eyes1, down_ratio, chin_width_ratio)
+            if face_direction == "left":
+                print("Left eye detected")
+                face_min_x = eyes1[0, 0]
+                face_max_x = min(img_width, int(eyes1[0,0] + (chin_width_ratio + 0.5) * eyes1[0, 2]))
+                forehead_max_x = min(img_width, int(eyes1[0,0] + width_ratio * eyes1[0, 2]))
+                forehead_min_x = face_min_x
+                cheek_min_x = int(eyes1[0, 0] + 0.5 * eyes1[0,2])
+                cheek_max_x = face_max_x
+            else:
+                print("Right eye detected")
+                face_min_x = max(0, int(eyes1[0, 0] - chin_width_ratio * eyes1[0, 2]))
+                face_max_x = eyes1[0, 0] + eyes1[0, 2]
+                forehead_min_x = max(0, int(eyes1[0, 0] - width_ratio * eyes1[0, 2]))
+                forehead_max_x = min(img_width, int(eyes1[0, 0] + width_ratio * eyes1[0, 2]))   
+                cheek_max_x = int(eyes1[0,0] + 0.5*eyes1[0,2])
+                cheek_min_x = face_min_x
+            forehead_min_y = max(0, int(eyes1[0, 1] - top_ratio * eyes1[0,3]))
+            forehead_max_y = max(0, int(eyes1[0, 1] - 0.5 * eyes1[0, 3]))
+            forehead_ok = False
+            # Get the forehead region
+            if forehead_max_y - forehead_min_y >= 0.7 * eyes1[0, 3]:
+                forehead_ok = True
+                forehead_region = img[forehead_min_y:forehead_max_y, forehead_min_x: forehead_max_x, :]
+                #print ('forehead dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (forehead_min_x,  forehead_max_x,  forehead_min_y,  forehead_max_y))
+                key_name = 'oneeye_fh'
+                dimension_dict[key_name] = [forehead_min_x,  forehead_max_x,  forehead_min_y,  forehead_max_y]
+                forehead_file_name = join(croppedFaces_Dir, key_name +".jpg")
+                imageio.imwrite(forehead_file_name, forehead_region)
+            # Get the cheek region
+            cheek_min_y = int(eyes1[0, 1] + eyes1[0, 3])
+            cheek_max_y = min(img_height, int(eyes1[0, 1] + down_ratio * eyes1[0, 3]))
+            cheek_region = img[cheek_min_y: cheek_max_y, cheek_min_x: cheek_max_x, :]
+            #print ('cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (cheek_min_x, cheek_max_x, cheek_min_y, cheek_max_y))
+            key_name = 'oneeye_cheek'
+            dimension_dict[key_name] = [cheek_min_x, cheek_max_x, cheek_min_y, cheek_max_y]
+            face_loc_dict['face_loc'] = [face_min_x, face_max_x, forehead_min_y, cheek_max_y]
+            #cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB)
+            if face_direction == "left":
+                cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
+            elif face_direction == "right":
+                cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
+            else:
+                cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
+            imageio.imwrite(cheek_file_name, cheek_region)
+    
+    
+    if (not face_detected) and (not eye_detected):
+        print("No chin or forehead detected, output the original file %s.jpg"%imageName)
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+        outfile = join(croppedFaces_Dir, imageName+".jpg")
+        imageio.imwrite(outfile, img)
+    
+    return dimension_dict, face_loc_dict, image_dim

index.py

@@ -0,0 +1,190 @@
+import json
+import get_patches
+import model
+from PIL import Image
+import numpy as np
+import os
+import shutil
+import openai
+from dotenv import load_dotenv
+load_dotenv()
+
+def convert_image(file_path):
+    img = Image.open(file_path)
+    img_re = img.resize((128, 128))
+    numpy_arr = np.asarray(img_re)
+    return numpy_arr
+
+def predict(image):
+    def extract_patches(image):
+        patch_path = 'patches'
+        dimension_dict = dict()
+        face_dict = dict()
+        image_dim = []
+        # try:
+        dim, face, img = get_patches.extract_patches(
+            image, dimension_dict, face_dict, image_dim, patch_path)
+        print("extract patches pass")
+        return dim, face, img
+        # except:
+        #     print("extract patches fail")
+        #     return None, None, None
+
+    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+    os.mkdir(os.path.join(BASE_DIR, 'patches'))
+
+    dim, face, img = extract_patches(image)
+
+    if dim is None and face is None and img is None:
+        return json.dumps({"msg": "fail"})
+
+    model_path = os.path.join(
+        BASE_DIR, 'Acne_Classifyer_N_Resnet.h5')
+
+    resnet_model = model.load_trained_model(model_path)
+    imageFiles = [os.path.join(BASE_DIR, 'patches', f)
+                  for f in os.listdir(os.path.join(BASE_DIR, 'patches'))]
+
+    images = []
+    landmarks = []
+    for f in imageFiles:
+        if f.endswith('.jpg'):
+            landmark = f.split('/')[-1].split('_')[-1]
+            landmarks.append(landmark)
+            images.append(convert_image(f))
+
+    images = np.array(images)
+
+    prediction = resnet_model.predict(images)
+    prediction = prediction.tolist()
+    shutil.rmtree(os.path.join(BASE_DIR, 'patches'))
+    return {"msg": "success", "prediction": prediction, "landmarks": landmarks}
+
+
+def recommendation(response):
+    openai.api_key = os.getenv("OPENAI_API_KEY")
+    json = response
+
+    if json.get('msg') == "success":
+        prediction = json.get('prediction')
+        prediction = np.array(prediction)
+        prediction = prediction.argmax(axis=1)
+
+        landmarks = json.get('landmarks')
+
+        output = {landmarks[i].split(
+            '.')[0]: f"level{prediction[i]}" for i in range(len(landmarks))}
+        output['age'] = json.get('age')
+        output['gender'] = json.get('gender')
+        output['skin_type'] = json.get('skin_type')
+        output['allergies'] = json.get('allergies')
+        output['diet'] = json.get('diet')
+        completion = openai.ChatCompletion.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "system", "content": "you are a medical AI assistant that assists patients with acne on face with getting rid of it, okay? don't give me tips just nod"},
+
+                {"role": "user", "content": "what do you know about acne severity levels like level 0, level 1 and level 2 ?"},
+
+                {"role": "assistant", "content": """Acne severity levels are used to classify the severity of acne, with level 0 being the mildest and level 4 being the most severe. However, the commonly used system usually consists of four levels, which are described as follows:
+                Level 0: This level is used to describe skin with no signs of acne. The skin is generally clear with no visible pimples or blemishes.
+                Level 1: This level is used to describe skin with minor acne. The skin may have some blackheads and whiteheads, but they are not widespread and there are no visible pimples.
+                Level 2: This level is used to describe skin with moderate acne. The skin may have more blackheads and whiteheads, as well as a few visible pimples.
+                Level 3: This level is used to describe skin with severe acne. The skin may have many blackheads, whiteheads, and visible pimples, as well as some nodules and cysts.
+                It is worth noting that acne severity can vary greatly from person to person, and there may be other factors to consider when assessing the severity of acne, such as the presence of scarring or inflammation. Therefore, it is important to consult with a dermatologist or healthcare professional for an accurate diagnosis and personalized treatment plan."""},
+
+                {"role": "user", "content": "okay let me teach u something new"},
+
+                {"role": "assistant", "content": "Sure, I'm always open to learning something new! What would you like to teach me?"},
+
+                {"role": "user", "content": """so level 0 is This level is used to describe skin with no signs of acne. The skin is generally clear with no visible pimples or blemishes.
+                level 1 This level is used to describe skin with minor acne. The skin may have some blackheads and whiteheads, but they are not widespread and there are no visible pimples.
+                level 2 This level is used to describe skin with moderate acne to severe acne. The skin may have few to many blackheads and whiteheads, as well as a few or many visible pimples, as well as some nodules and cysts. there does not exist any other level these 3 are the only levels you know about okay"""},
+
+                {"role": "assistant", "content": """I apologize for the error in my previous response. You are correct that the commonly used acne severity classification system consists of three levels, not four. Here is the correct information:
+                Level 0: This level is used to describe skin with no signs of acne. The skin is generally clear with no visible pimples or blemishes.
+                Level 1: This level is used to describe skin with mild acne. The skin may have some blackheads and whiteheads, but they are not widespread and there are no visible pimples.
+                Level 2: This level is used to describe skin with moderate to severe acne. The skin may have many blackheads and whiteheads, as well as a few or many visible pimples, as well as some nodules and cysts.
+                Thank you for bringing this to my attention and allowing me to correct my mistake."""},
+
+                {"role": "user", "content": "okay now i will tell you some instructions and the responses with boudries to which you will adhere"},
+
+                {"role": "assistant", "content": "Sure, please go ahead and give me the instructions and boundaries to adhere to."},
+
+                {"role": "user",
+                    "content": """so we have divided a face into patches these are namely fh(forehead), lc(left cheek), rc(right cheek), ch(chin). Each time my system will analyze the face image extract the patches from it and will send them to you along with some other info of the user which are Age, Gender(Male, Female, other), Skin-Type(Oily, Dry, Normal), Allergies(salicylic acid, benzoyl peroxide, Sun-exposure, Itching, Swelling, Redness), Diet(Veg, Non-veg) these are a total of 5 meta-data values about the person or individual for whom you will generate the Diet, Daily Skin-care Routine,  it is not necessary that each time you will be provided with all the 4 mentioned patches above(but you will always be provided with the 5 meta-data values of the individual)  some time the message may contain only 1 or 2 patches. I will also send the acne severity level associated with that patch like {'fh':'level0', 'lc':'level2', 'rc':'level2', 'chin':'level0', 'age':20, 'gender':'male', 'skin':'oily', 'allergies':['itching'], 'diet':'veg'} this json data means that the patch forehead has level0 acne and left cheek has level2 acne and he is of age 20 and he has oily skin and have itching allergy and his diet is veg. Now that you now what level of acne does the parts of a particular face has you will generate a week long custom made remedy, diet, good habits, bad habits and daily skincare routine for the acne treatment. I can ask you to generate a general acne routine and remedy but i am providing you with this classified details so that the output you will generate will be more optimized and personalized. okay that's it if you understand completely what i just instructed to you then generate a sample response for me for the above example data. also next time i will not be sending such long detailed message just the json message that i sent above so you will not get confused and do the task as i told you. also the response should be in a raw mark up language format the language should be raw html with <br> tag at places where line break is necessary so i can copy it as it is and use it some where else. Also i need you to generate answers in way that they don't sound like they are being generated by a chat bot, leave out any sentences like "Sure, here's a customized routine for the given example:" so you need to sound like the answers are being generated by a deep learning NLP model not by a chat bot. You don't need to greet to me in the response just straight out tell the answer don't greet to me like 'here is the routine for the example' or anything like that."""},
+
+                {"role": "assistant", "content": """
+                <h3>Skincare Routine:</h3>
+                <ul>
+                <li>Wash your face with a gentle cleanser twice a day.</li>
+                <li>Use an alcohol-free toner after cleansing to balance your skin.</li>
+                <li>Apply a moisturizer with SPF in the morning.</li>
+                <li>Apply a retinoid cream at night to help unclog pores.</li>
+                <li>Use a clay mask once a week to draw out impurities.</li>
+                </ul>
+                <h3>Diet:</h3>
+                <p>Since the individual has oily skin, it's recommended to follow a low-glycemic index diet which includes more fruits, vegetables, whole grains, and lean proteins. Also, since they have a vegetarian diet, they can consume more nuts, seeds, and legumes as sources of protein. It's also important to drink plenty of water and avoid sugary and processed foods as they can cause inflammation in the body and lead to acne breakouts.</p>
+                <h3>Good Habits:</h3>
+                <ul>
+                <li>Keep your hands off your face to prevent bacteria transfer.</li>
+                <li>Change your pillowcases regularly to avoid buildup of oils and bacteria.</li>
+                <li>Avoid touching or picking at acne as it can lead to scarring.</li>
+                <li>Wash your makeup brushes regularly to avoid buildup of bacteria.</li>
+                <li>Wear sunscreen every day to protect your skin from harmful UV rays.</li>
+                </ul>
+                <h3>Bad Habits:</h3>
+                <ul>
+                <li>Avoid smoking as it can cause premature aging and damage to the skin.</li>
+                <li>Avoid excessive alcohol consumption as it can dehydrate the skin.</li>
+                <li>Avoid using harsh scrubs or exfoliants as they can irritate the skin.</li>
+                <li>Avoid using products with salicylic acid or benzoyl peroxide if the individual has an allergy to them.</li>
+                <li>Avoid consuming processed and sugary foods as they can cause inflammation and lead to acne breakouts.</li>
+                </ul>
+                <h3>Additional Recommendations:</h3>
+                <p>Since the individual has an itching allergy, it's recommended to avoid products with fragrance and other potential irritants. Also, since they have oily skin, it's recommended to use non-comedogenic and oil-free products to avoid clogging pores.</p>"""},
+
+                                {"role": "user",
+                                    "content": "{'fh':'level2', 'lc':'level1', 'rc':'level0', 'chin':'level2', 'age':29, 'gender':'female', 'skin':'dry', 'allergies':['itching', 'swelling', 'sun-exposure', 'redness'], 'diet':'non-veg'"},
+
+                                {"role": "assistant", "content": """
+                                <h3>Week-long Acne Treatment Plan</h3>
+                <p>Age: 29 | Gender: Female | Skin Type: Dry | Allergies: Itching,Swelling,Sun-exposure,Redness | Diet: Non-Veg</p>
+                <h4>Daily Skincare Routine:</h4>
+                <ul>
+                <li>Wash your face twice a day using a gentle, non-comedogenic cleanser.</li>
+                <li>Apply a moisturizer with SPF 30 or higher to protect your skin from sun damage.</li>
+                <li>Use a toner containing salicylic acid to help unclog pores and reduce acne.</li>
+                <li>Apply a spot treatment containing benzoyl peroxide to acne-prone areas.</li>
+                </ul>
+                <h4>Diet Recommendations:</h4>
+                <ul>
+                <li>Include foods rich in vitamin A, such as sweet potatoes, carrots, and leafy greens, which can help reduce inflammation and promote healthy skin.</li>
+                <li>Avoid dairy products and high-glycemic foods, such as white bread and sugary snacks, which may worsen acne.</li>
+                <li>Incorporate foods rich in omega-3 fatty acids, such as salmon and walnuts, which can help reduce inflammation and improve skin health.</li>
+                </ul>
+                <h4>Good Habits:</h4>
+                <ul>
+                <li>Get enough sleep each night to help reduce stress, which can trigger acne.</li>
+                <li>Drink plenty of water to stay hydrated and flush out toxins.</li>
+                <li>Exercise regularly to help reduce stress and improve overall health.</li>
+                </ul>
+                <h4>Bad Habits to Avoid:</h4>
+                <ul>
+                <li>Avoid touching your face, as this can transfer oil and bacteria to the skin and worsen acne.</li>
+                <li>Avoid using harsh scrubs or exfoliants, which can irritate the skin and worsen acne.</li>
+                <li>Avoid picking or squeezing pimples, as this can lead to scarring and further breakouts.</li>
+                </ul>
+                <h4>Acne Treatment:</h4>
+                <p>Based on the severity of acne on different patches of the face:</p>
+                <ul>
+                <li>For level 0 acne on fh, chin: continue with daily skincare routine and good habits.</li>
+                <li>For level 1 acne on lc: use an over-the-counter retinoid cream in the evening to help unclog pores and reduce acne.</li>
+                <li>For level 2 acne on fh, rc, chin: consult a dermatologist for prescription-strength topical or oral medications.</li>
+                </ul>"""},
+                {"role": "user", "content": f"{output}"},
+            ]
+        )
+
+        return completion
+    return ""

model.py

@@ -0,0 +1,36 @@
+import tensorflow as tf
+import os
+
+resnet101 = tf.keras.applications.ResNet101(include_top=False, weights="imagenet")
+
+for layer in resnet101.layers:
+  layer.trainable = False
+  if layer.name == "conv3_block1_1_conv":
+    break
+
+IMG_HEIGHT = 128
+IMG_WIDTH = 128
+IMG_CHANNELS = 3
+
+def load_trained_model(weights_path):
+    input = tf.keras.layers.Input(shape=(IMG_WIDTH, IMG_HEIGHT, IMG_CHANNELS))
+
+    # data augmentation
+    a1 = tf.keras.layers.experimental.preprocessing.RandomRotation(0.4)(input)
+    a2 = tf.keras.layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical")(a1)
+
+    # resnet
+    out = resnet101(a2)
+    x = tf.keras.layers.GlobalMaxPooling2D()(out)
+
+    # dense layer
+    d1 = tf.keras.layers.Dense(256, activation="relu")(x)
+    d2 = tf.keras.layers.Dense(128, activation="relu")(d1)
+    d3 = tf.keras.layers.Dense(64, activation="relu")(d2)
+    prediction = tf.keras.layers.Dense(3, activation="softmax")(d3)
+
+    model = tf.keras.Model(inputs=input, outputs=prediction)
+    model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
+
+    model.load_weights(weights_path)
+    return model

requirements.txt

@@ -0,0 +1,111 @@
+absl-py==1.4.0
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+astunparse==1.6.3
+async-timeout==4.0.2
+attrs==22.2.0
+autopep8==2.0.2
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+dlib==19.24.1
+entrypoints==0.4
+fastapi==0.95.0
+ffmpy==0.3.0
+filelock==3.11.0
+Flask==2.2.3
+flatbuffers==23.3.3
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.3.0
+gast==0.4.0
+google-auth==2.17.2
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+gradio==3.24.1
+gradio_client==0.0.8
+grpcio==1.53.0
+h11==0.14.0
+h5py==3.8.0
+httpcore==0.16.3
+httpx==0.23.3
+huggingface-hub==0.13.4
+idna==3.4
+imageio==2.27.0
+itsdangerous==2.1.2
+jax==0.4.8
+Jinja2==3.1.2
+jsonschema==4.17.3
+keras==2.12.0
+kiwisolver==1.4.4
+lazy_loader==0.2
+libclang==16.0.0
+linkify-it-py==2.0.0
+Markdown==3.4.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+ml-dtypes==0.0.4
+multidict==6.0.4
+networkx==3.1
+numpy==1.23.5
+oauthlib==3.2.2
+openai==0.27.4
+opencv-python==4.7.0.72
+opt-einsum==3.3.0
+orjson==3.8.9
+packaging==23.0
+pandas==2.0.0
+Pillow==9.5.0
+protobuf==4.22.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.10.0
+pydantic==1.10.7
+pydub==0.25.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-multipart==0.0.6
+pytz==2023.3
+PyWavelets==1.4.1
+PyYAML==6.0
+requests==2.28.2
+requests-oauthlib==1.3.1
+rfc3986==1.5.0
+rsa==4.9
+scikit-image==0.20.0
+scipy==1.10.1
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.26.1
+tensorboard==2.12.1
+tensorboard-data-server==0.7.0
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.12.0
+tensorflow-estimator==2.12.0
+tensorflow-io-gcs-filesystem==0.32.0
+termcolor==2.2.0
+tifffile==2023.3.21
+tomli==2.0.1
+toolz==0.12.0
+tqdm==4.65.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.1
+urllib3==1.26.15
+uvicorn==0.21.1
+websockets==11.0.1
+Werkzeug==2.2.3
+wrapt==1.14.1
+yarl==1.8.2