First Trial

models.py

@@ -0,0 +1,459 @@
+# keras vggface model
+import tensorflow as tf
+from keras.layers import Flatten, Dense, Input, Dropout, Activation, BatchNormalization
+
+from keras_vggface.vggface import VGGFace
+from keras.models import Model
+# example of loading an image with the Keras API
+# since 2021 tensorflow updated the package and moved model directory
+from tensorflow.keras.preprocessing import image
+import keras_vggface.utils as utils
+
+# image manipulation
+from matplotlib import pyplot as plt
+import matplotlib.patches as patches
+from PIL import Image, ImageDraw
+import cv2
+
+# face alignment
+from mtcnn.mtcnn import MTCNN
+
+# model metrics
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics import mean_squared_error, mean_absolute_error
+from scipy.stats import pearsonr
+
+# common packages
+import os
+import numpy as np
+import pandas as pd
+import pickle
+
+import shutil
+from tqdm import tqdm
+import tempfile
+import hashlib
+
+# Operations regarding to folder/file
+def copy_images(file_paths, source_folder, destination_folder):
+    for file_path in file_paths:
+        file_name = os.path.basename(file_path)
+        source_file = os.path.join(source_folder, file_name)
+        destination_file = os.path.join(destination_folder, file_name)
+        shutil.copyfile(source_file, destination_file)
+
+def get_file_names(folder_path):
+    file_names = []
+    for file_name in os.listdir(folder_path):
+        if os.path.isfile(os.path.join(folder_path, file_name)):
+            file_names.append(file_name)
+    return file_names
+
+# Easy-to-use Performance metrics
+def rmse(x,y):
+    return np.sqrt(mean_squared_error(x,y))
+
+def mae(x,y):
+    return mean_absolute_error(x,y)
+
+def auc(label, pred):
+    return roc_auc_score(label, pred)
+
+
+# Previous codes for image2array processing; still adopted for single imgae prediction
+def imgs_to_array(img_paths, version=1):
+    ''' extract features from all images and convert to multi-dimensional array
+    Takes:
+        img_path: str
+        version: int
+    Returns:
+        np.array
+    '''
+    imgs = []
+    for img_path in img_paths: # += is equivalent to extend @http://noahsnail.com/2020/06/17/2020-06-17-python%E4%B8%ADlist%E7%9A%84append,%20extend%E5%8C%BA%E5%88%AB/
+        imgs += [img_to_array(img_path, version)]
+    return np.concatenate(imgs)
+
+def process_array(arr, version):
+    '''array processing (resize)
+    Takes: arr: np.array
+    Returns: np.array
+    '''
+    desired_size = (224, 224)
+    img = cv2.resize(arr, desired_size)
+    img = img * (1./255)
+    #img = np.expand_dims(img, axis=0)
+    img = utils.preprocess_input(img, version=version)
+    return img
+
+def img_to_array(img_path, version):
+    '''conver a SINGLE image to array
+    Takes: img_path: str
+    Returns: np.array
+    '''
+    if not os.path.exists(img_path):
+        return None  
+
+    img = image.load_img(img_path)
+    img = image.img_to_array(img)
+    img = process_array(img, version)
+    return img
+
+def crop_img(img,x,y,w,h):
+    '''crop image
+    Takes: img: np.array
+           x,y,w,h: int
+    Returns: np.array
+    '''
+    return img[y:y+h,x:x+w,:]
+
+def array_to_img(arr):
+    '''Converts a numpy array to an image.
+    Takes: arr: np.array
+    Returns: PIL.Image
+    '''
+    # Convert array to image
+    img = Image.fromarray(np.uint8(arr*255))
+    return img
+
+
+# build a ImageDataGenerator
+from tensorflow.keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
+
+def single_test_generator(img_path, target_size=(224, 224), batch_size=1):
+    '''Generate a single test generator from an image file.
+    Takes:
+        - img_path: str, path to the image file
+        - target_size: tuple, target size for image resizing (default: (224, 224))
+        - batch_size: int, batch size for the generator (default: 32)
+    Returns:
+        - single_test_gen: ImageDataGenerator, generated image generator
+    '''
+    # Load the image
+    img = load_img(img_path, target_size=target_size)
+    # Convert the image to an array
+    img_array = img_to_array(img)
+    # Reshape the array to match the expected input shape of the model
+    img_array = img_array.reshape((1,) + img_array.shape)
+    # Create an instance of ImageDataGenerator
+    test_datagen = ImageDataGenerator(
+        rescale = 1./255,
+        shear_range = 0.2,
+        zoom_range = 0.2,
+        horizontal_flip = True)
+    # Generate the images
+    single_test_gen = test_datagen.flow(img_array, batch_size=batch_size)
+
+    return single_test_gen
+
+# Create the ImageDataGenerator for the test_data
+test_datagen = ImageDataGenerator(
+    #preprocessing_function=lambda x: (x - mean_pixel_value) / 255.0,
+    rescale = 1./255)
+
+def img_data_generator(data, bs, img_dir, train_mode=True, version = 1): #replace function name later
+    """data input pipeline
+    Takes:
+        data: pd.DataFrame
+        bs: batch size
+        img_dir: str, directory to the images
+        train_mode: bool, if False, take samples from test set to aoivd overfitting
+        version: int, keras_vggface version
+    Returns:
+        features: tuple of (x,y): features and targets
+    """
+    loop = True
+     
+    while loop:
+        if train_mode:
+            x = imgs_to_array(data['path'], version)
+            y = data['bmi'].values
+            features = (x,y)
+        else:
+            if len(data) >= bs:
+                sampled = data.iloc[:bs,:]
+                data = data.iloc[bs:,:]
+                features = imgs_to_array(sampled['index'], img_dir, version)
+            else: 
+                loop = False
+        yield features
+
+
+
+# Build a prediction class
+class FacePrediction(object):
+
+    def __init__(self, img_dir=None, model_type='vgg16'):
+        self.model_type = model_type
+        self.img_dir = img_dir
+        self.detector = MTCNN()
+        if model_type in ['vgg16', 'vgg16_fc6']: # we might use other models, but in that case we need to just version input
+            self.version = 1
+        else:
+            self.version = 2
+
+    def define_model(self, hidden_dim = 64, drop_rate=0.0, freeze_backbone = True): # replace function name later
+        ''' initialize the vgg model
+        Reference:
+            @https://zhuanlan.zhihu.com/p/53116610
+            @https://zhuanlan.zhihu.com/p/26934085
+        '''
+        if self.model_type == 'vgg16_fc6':
+            vgg_model = VGGFace(model = 'vgg16', include_top=True, input_shape=(224, 224, 3))
+            last_layer = vgg_model.get_layer('fc6').output
+            flatten = Activation('relu')(last_layer)
+        else:
+            vgg_model = VGGFace(model = self.model_type, include_top=False, input_shape=(224, 224, 3))
+            last_layer = vgg_model.output
+            flatten = Flatten()(last_layer)
+        
+        if freeze_backbone: # free the vgg layers to fine-tune
+            for layer in vgg_model.layers:
+                layer.trainable = False
+                
+        def model_init(flatten, name):
+            # x = Dense(64, name=name + '_fc1')(flatten)
+            # x = BatchNormalization(name = name + '_bn1')(x)
+            # x = Activation('relu', name = name+'_act1')(x)
+            # x = Dropout(0.2)(x)
+            # x = Dense(64, name=name + '_fc2')(x)
+            # x = BatchNormalization(name = name + '_bn2')(x)
+            # x = Activation('relu', name = name+'_act2')(x)
+            # x = Dropout(drop_rate)(x)
+            x = flatten
+            return x
+        
+        x = model_init(flatten, name = 'bmi')
+        bmi_pred = Dense(1, activation='linear', name='bmi')(x) #{'relu': , 'linear': terrible}
+
+        custom_vgg_model = Model(vgg_model.input, bmi_pred)
+        custom_vgg_model.compile('adam', 
+                                 {'bmi':'mae'}, #{'bmi':'mae'},
+                                 loss_weights={'bmi': 1})
+
+        self.model = custom_vgg_model
+
+    def train(self, train_gen, val_gen, train_step, val_step, bs, epochs, callbacks):
+        ''' train the model
+        Takes: 
+            train_data: dataframe
+            val_data: dataframe
+            bs: int, batch size
+            epochs: int, number of epochs
+            callbacks: list, callbacks
+        Recall the input for img_data_generator: data, bs, img_dir, train_mode=True, version = 1
+        '''
+        self.model.fit_generator(train_gen, train_step, epochs=epochs,
+                                 validation_data=val_gen, validation_steps=val_step,
+                                 callbacks=callbacks)
+
+
+    def evaluate_perf(self, val_data):
+        img_paths = val_data['path'].values
+        arr = imgs_to_array(img_paths, self.version)
+        bmi = self.model.predict(arr)
+        metrics = {'bmi_mae':mae(bmi[:,0], val_data.bmi.values)}
+        return metrics
+    
+    def detect_faces(self, img_path, confidence):
+        img = image.load_img(img_path)
+        img = image.img_to_array(img)
+        box = self.detector.detect_faces(img)
+        box = [i for i in box if i['confidence'] > confidence]
+        res = [crop_img(img, *i['box']) for i in box]
+        res = [process_array(i, self.version) for i in res]
+        return box, res
+
+    def crop_image_around_face(self, img, box, crop_percentage):
+        x, y, width, height = box['box']
+        center_x = x + (width // 2)
+        center_y = y + (height // 2)
+        crop_width = int(width * crop_percentage)
+        crop_height = int(height * crop_percentage)
+        crop_left = max(0, center_x - (crop_width // 2))
+        crop_top = max(0, center_y - (crop_height // 2))
+        crop_right = min(img.width, crop_left + crop_width)
+        crop_bottom = min(img.height, crop_top + crop_height)
+        cropped_img = img.crop((crop_left, crop_top, crop_right, crop_bottom))
+        return cropped_img
+
+    def process_input_image(self, img_input_path):
+        img = Image.open(img_input_path)
+
+        # Check image size
+        if img.size == (244, 244):
+            return img_input_path
+        else:
+            # Detect faces and crop
+            confidence_threshold = 0.5
+            boxes, cropped_images = self.detect_faces(img_input_path, confidence_threshold)
+
+            if len(cropped_images) > 0:
+                # Save the cropped image in a temporary folder
+                tmp_folder = 'tmp'
+                os.makedirs(tmp_folder, exist_ok=True)
+
+                # Generate hash value from the image input path
+                hash_value = hashlib.sha1(img_input_path.encode()).hexdigest()
+
+                tmp_img_path = os.path.join(tmp_folder, hash_value + 'temp_image.bmp')
+
+                # Print confidence for each detected face
+                for i, box in enumerate(boxes):
+                    confidence = box['confidence']
+                    print(f"Face {i + 1}: Confidence - {confidence}")
+
+                    # Crop the image around the detected face
+                    cropped_img = self.crop_image_around_face(img, box, crop_percentage=1.25)
+
+                # Save the cropped image
+                cropped_img.save(tmp_img_path)
+
+                return tmp_img_path
+            else:
+                # No faces detected, return the original image
+                return img_input_path
+
+    def predict_external(self, img_input_dir, input_df=None, image_width=244, image_height=244, batch_size=32, show_img=False):
+        if os.path.isdir(img_input_dir) and input_df is not None:
+            # Predict using the data generator
+            test_df = input_df
+            processed_img_paths = [self.process_input_image(i) for i in test_df['path']]
+            processed_img_names = [i.split('/')[-1] for i in processed_img_paths]
+            processed_img_dir = '/'.join(processed_img_paths[0].split('/')[:-1])
+            test_df['processed_paths'], test_df['processed_names'] = processed_img_paths, processed_img_names
+            
+
+            # Load the test data with target data
+            test_set_gen = test_datagen.flow_from_dataframe(
+                test_df,
+                directory = img_input_dir,
+                x_col='name',
+                y_col='bmi',
+                target_size=(image_width, image_height),
+                batch_size=batch_size,
+                color_mode='rgb',
+                class_mode='raw')
+
+            preds = self.model.predict_generator(test_set_gen)
+
+            if show_img and (test_df is not None):
+                bmi = preds
+                num_plots = len(test_df['path'])
+                ncols = 5
+                nrows = int((num_plots - 0.1) // ncols + 1)
+                fig, axs = plt.subplots(nrows, ncols)
+                fig.set_size_inches(3 * ncols, 3 * nrows)
+                for i, img_path in enumerate(test_df['path']):
+                    col = i % ncols
+                    row = i // ncols
+                    img = plt.imread(img_path)
+                    axs[row, col].imshow(img)
+                    axs[row, col].axis('off')
+                    axs[row, col].set_title('BMI: {:3.1f}'.format(bmi[i, 0], fontsize=10))
+            return preds
+
+        else:
+            # Single image input
+            single_test_path = self.process_input_image(img_input_dir)
+            single_test_gen = single_test_generator(single_test_path)
+
+            if show_img:
+                img_path = img_input_dir
+                img = plt.imread(single_test_path)
+                fig, ax = plt.subplots()
+                ax.imshow(img)
+                ax.axis('off')
+                preds = self.model.predict_generator(single_test_gen)
+                ax.set_title('BMI: {:3.1f}'.format(preds[0, 0], fontsize=10))
+                plt.show()
+
+            preds = self.model.predict_generator(single_test_gen)
+            return preds
+
+
+
+    def predict(self, img_input_dir, input_generator=None, input_df=None, show_img=False):
+        if os.path.isdir(img_input_dir) and input_generator is not None:
+            # Predict using the data generator
+            preds = self.model.predict_generator(input_generator)
+
+            if show_img and (input_df is not None):
+                bmi = preds
+                num_plots = len(input_df['path'])
+                ncols = 5
+                nrows = int((num_plots - 0.1) // ncols + 1)
+                fig, axs = plt.subplots(nrows, ncols)
+                fig.set_size_inches(3 * ncols, 3 * nrows)
+                for i, img_path in enumerate(input_df['path']):
+                    col = i % ncols
+                    row = i // ncols
+                    img = plt.imread(img_path)
+                    axs[row, col].imshow(img)
+                    axs[row, col].axis('off')
+                    axs[row, col].set_title('BMI: {:3.1f}'.format(bmi[i, 0], fontsize=10))
+            return preds
+
+        else:
+            single_test_gen = single_test_generator(img_input_dir)
+            
+            if show_img:
+                img_path = img_input_dir
+                img = plt.imread(img_path)
+                fig, ax = plt.subplots()
+                ax.imshow(img)
+                ax.axis('off')
+                #preds = self.model.predict(img_to_array(img_path, self.version))
+                preds = self.model.predict_generator(single_test_gen)
+                ax.set_title('BMI: {:3.1f}'.format(preds[0, 0], fontsize=10))
+                plt.show()
+
+            preds = self.model.predict_generator(single_test_gen)
+            #preds = self.model.predict(img_to_array(img_path, self.version))
+            return preds
+
+
+    def predict_df(self, img_dir):
+        assert os.path.isdir(img_dir), 'input must be directory'
+        fnames = os.listdir(img_dir)
+        bmi = self.predict(img_dir)
+        results = pd.DataFrame({'img':fnames, 'bmi':bmi[:,0]})
+        return results
+    
+    def save_weights(self, model_dir):
+        self.model.save_weights(model_dir)
+
+    def load_weights(self, model_dir):
+        self.model.load_weights(model_dir)
+
+    def load_model(self, model_dir):
+        self.model.load_model(model_dir)
+
+    def predict_faces(self, img_path, show_img = True, color = "white", fontsize = 12, 
+                      confidence = 0.95, fig_size = (16,12)):
+
+        assert os.path.isfile(img_path), 'only single image is supported'
+        img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
+        boxes, faces = self.detect_faces(img_path, confidence)
+        preds = [self.model.predict(face) for face in faces]
+
+        if show_img:
+            # Create figure and axes
+            num_box = len(boxes)
+            fig,ax = plt.subplots()
+            fig.set_size_inches(fig_size)
+            # Display the image
+            ax.imshow(img)
+            ax.axis('off')
+            # Create a Rectangle patch
+            for idx, box in enumerate(boxes):
+                bmi = preds[idx]
+                box_x, box_y, box_w, box_h = box['box']
+                rect = patches.Rectangle((box_x, box_y), box_w, box_h, linewidth=1,edgecolor='yellow',facecolor='none')
+                ax.add_patch(rect)
+                ax.text(box_x, box_y, 
+                        'BMI:{:3.1f}'.format(bmi[0,0]),
+                       color = color, fontsize = fontsize)
+            plt.show()
+
+        return preds

models/0508_saved_model_small_sample_vgg16_base.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9b29ebadad26dc628dc191d81dbd91669094dcac4d196e4c91a8cdcc5c7132
+size 59239768