In [1]:
# import libraries
import numpy as np
import tensorflow as tf
import keras_tuner
import PIL
import matplotlib.pyplot as plt
import pandas as pd
import os
# Seed the tf, numpy and python with a fix random number
seed = 93
import random # from the python library
random.seed(seed)
np.random.seed(seed)
tf.random.set_seed(seed)
# Various directories
CHECKPOINT_DIR = os.path.join("checkpoint", "all_multi_tuner")
LOG_DIR = os.path.join("logdir", "all_multi_tuner")
HP_DIR = os.path.join("hp_search", "all_multi_tuner")
MODEL_TF_DIR = os.path.join("model", "all_multi_tuner")
MODEL_KERAS_DIR = os.path.join("model", "all_multi_tuner", "multi_bin_acc.keras")
# Set the difficulty level to train
DIFF_LEVEL = "multi" # binary, multi, easy, mid, hard
# Train all the images with binary label
NUM_CLASSES = 5 # number of classes = 5 if multilabel is used
In [3]:
# The various directories
print(f"Checkpoint: {CHECKPOINT_DIR}")
print(f"Log: {LOG_DIR}")
print(f"Hyperparamters: {HP_DIR}")
print(f"TF model: {MODEL_TF_DIR}")
print(f"Keras model: {MODEL_KERAS_DIR}")
Checkpoint: checkpoint\all_multi_tuner Log: logdir\all_multi_tuner Hyperparamters: hp_search\all_multi_tuner TF model: model\all_multi_tuner Keras model: model\all_multi_tuner\multi_bin_acc.keras
In [4]:
# List the gpu
devices = tf.config.list_physical_devices("GPU")
print(devices)
[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
In [5]:
# path of the dataset saved from previous step
# path: dataset\train\ds_binary.
# path: dataset\train\ds_multi.
diff_level = f"ds_{DIFF_LEVEL}"
dataset_file = os.path.join("dataset", "train", diff_level)
print(f"Dataset file: {dataset_file}")
if os.path.exists(dataset_file) is False:
print(f"File not found: {dataset_file}")
exit(0)
# Load the dataset
ds = tf.data.Dataset.load(dataset_file)
Dataset file: dataset\train\ds_multi
In [6]:
# print the loaded dataset info
def print_dataset_info(dataset):
print(f"Image arrays: {dataset.element_spec[0].shape}, {dataset.element_spec[0].dtype}")
print(f"Label arrays: {dataset.element_spec[1].shape}, {dataset.element_spec[1].dtype}")
print(f"cardinality: {dataset.cardinality()}")
In [7]:
print_dataset_info(ds)
Image arrays: (224, 224, 3), <dtype: 'float32'> Label arrays: (5,), <dtype: 'int32'> cardinality: 2028
Prepare dataset¶
In [8]:
# split the dataset into train and validation set
ds_train, ds_val = tf.keras.utils.split_dataset(
ds, left_size=0.9, right_size=0.1, shuffle=True, seed=seed
)
In [9]:
print(f"train set: {ds_train.cardinality()}, \nvalidation set: {ds_val.cardinality()}")
train set: 1825, validation set: 203
In [10]:
# preprocess input
# data augmentation and the pretrained model's preprocess function
img_augmentation_layers = [
tf.keras.layers.RandomRotation(factor=0.15),
tf.keras.layers.RandomTranslation(height_factor=0.1, width_factor=0.1),
]
def img_augmentation(images):
for layer in img_augmentation_layers:
images = layer(images)
return images
# the preprocess function of the pretrained model
pretrained_model_preprocess_fn = tf.keras.applications.efficientnet_v2.preprocess_input
def input_preprocess_train(image, label):
#image = img_augmentation(image) # not used
image = pretrained_model_preprocess_fn(image)
return image, label
# do not augment the data in the validation or test set
def input_preprocess_test(image, label):
image = pretrained_model_preprocess_fn(image)
return image, label
In [11]:
BATCH_SIZE = 16
ds_train = ds_train.shuffle(buffer_size=1000, seed=seed)
ds_train = ds_train.map(lambda x, y: input_preprocess_train(x, y),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
ds_train = ds_train.cache()
ds_train = ds_train.batch(batch_size=BATCH_SIZE,
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True,
drop_remainder=True)
ds_train = ds_train.prefetch(tf.data.AUTOTUNE)
In [12]:
ds_val = ds_val.shuffle(buffer_size=1000, seed=seed)
ds_val = ds_val.map(lambda x, y: input_preprocess_test(x, y),
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True)
ds_val = ds_val.cache()
ds_val = ds_val.batch(batch_size=BATCH_SIZE,
num_parallel_calls=tf.data.AUTOTUNE,
deterministic=True,
drop_remainder=True)
ds_val = ds_val.prefetch(tf.data.AUTOTUNE)
In [13]:
ds_t = ds_train.take(1)
ds_v = ds_val.take(1)
In [14]:
ds_t
Out[14]:
<TakeDataset element_spec=(TensorSpec(shape=(16, 224, 224, 3), dtype=tf.float32, name=None), TensorSpec(shape=(16, 5), dtype=tf.int32, name=None))>
In [15]:
ds_v
Out[15]:
<TakeDataset element_spec=(TensorSpec(shape=(16, 224, 224, 3), dtype=tf.float32, name=None), TensorSpec(shape=(16, 5), dtype=tf.int32, name=None))>
In [16]:
print(f"train set: {ds_t.cardinality()}, \nvalidation set: {ds_v.cardinality()}")
train set: 1, validation set: 1
Callbacks layers and optimizer learning rate scheduler¶
In [17]:
# The ModelCheckpoint callback can be used to implement fault-tolerance: the
# ability to restart training from the last saved state of the model in case
# training gets randomly interrupted.
# setup the callbacks
# tensorboard log directory
logdir = LOG_DIR
checkpoint_dir = CHECKPOINT_DIR
# check for the directory
if not os.path.exists(logdir):
os.mkdir(logdir)
if not os.path.exists(checkpoint_dir):
os.mkdir(checkpoint_dir)
import time # to format time
model_name = f"best_{DIFF_LEVEL}_{time.strftime('%d_%m_%H%M')}.keras"
checkpoint_filepath = os.path.join(checkpoint_dir, model_name)
print(f"Checkpoint filepath: {checkpoint_filepath}")
callbacks = [
# early stopping
tf.keras.callbacks.EarlyStopping(patience=150),
# Save the best model
tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=True,
monitor='val_binary_accuracy',
mode='max',
save_best_only=True
),
# tensorboard
tf.keras.callbacks.TensorBoard(
log_dir=logdir,
update_freq="epoch"
)
]
Checkpoint filepath: checkpoint\all_multi_tuner\best_multi_11_02_1312.keras
Learning rate scheduluer¶
In [18]:
# Not using
# Gradually reduce learning rate
# Use default value
initial_learning_rate = 1e-3
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate,
decay_steps=100000,
decay_rate=0.96,
staircase=True
)
lr_schedule
Out[18]:
<keras.optimizers.schedules.learning_rate_schedule.ExponentialDecay at 0x1dcaa87b1f0>
In [ ]:
Efficient net¶
In [19]:
# prepare the dataset
def plot_history(history):
plt.plot(history.history["binary_accuracy"])
plt.plot(history.history["val_binary_accuracy"])
plt.title("Model accuracy")
plt.ylabel("accuracy")
plt.xlabel("epoch")
plt.grid(visible=True, axis="both")
plt.legend(["train", "validation"], loc="upper left")
plt.show()
In [20]:
# EfficientNetB7, image resolution: 600 x 600
# Get the image size
IMG_SIZE = ds.element_spec[0].shape[1]
print(f"Image size: {IMG_SIZE}")
Image size: 224
In [21]:
# EfficientNetV2 models expect their inputs to be float tensors of pixels with values in the [0-255] range.
# Use EfficientNetB7
# build_model: Set the range of hyperparameters.
# Then, it will call the exisiting model building code.
def build_model(hp):
pooling = hp.Choice("pooling", ["average", "global"])
norm = hp.Boolean("norm")
units = hp.Int("units", min_value=64, max_value=512, step=32)
dropout = hp.Float("dropout", min_value=0.1, max_value=0.4, step=0.1)
# call the actual model building code
model = call_existing_code(pooling, norm, units, dropout)
return model
# This is called by the keras tuner.
# This is the existing model building code that is working.
def call_existing_code(pooling, norm, units, dropout):
inputs = tf.keras.layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))
# drop_connect_rate which controls the dropout rate responsible for stochastic depth
# use this for stronger regularization
global base_model
base_model = tf.keras.applications.EfficientNetV2B0(
include_top=False,
input_tensor=inputs,
weights="imagenet",
)
# Freeze the pretrained weights
base_model.trainable = False
# Rebuild top
# pooling=None means that the output of the model will be the
# 4D tensor output of the last convolutional layer.
if pooling == "average":
x = tf.keras.layers.GlobalAveragePooling2D(name="avg_pool")(base_model.output)
elif pooling == "global":
x = tf.keras.layers.GlobalMaxPooling2D(name="max_pool")(base_model.output)
# if normalization is used
if norm is True:
x = tf.keras.layers.BatchNormalization()(x)
# Add a dense layer
# The units are chosen from the hyperparameters
x = tf.keras.layers.Dense(
units,
activation='relu',
kernel_regularizer="l2",
bias_regularizer="l2",
activity_regularizer="l2",
)(x)
# Drop out
# dropout rate from the hyperparameters
top_dropout_rate = dropout
x = tf.keras.layers.Dropout(
top_dropout_rate,
name="top_dropout",
seed=seed
)(x)
# set the number of classes
num_classes = NUM_CLASSES
outputs = tf.keras.layers.Dense(
num_classes,
activation="sigmoid",
name="predict",
kernel_regularizer="l2",
bias_regularizer="l2",
activity_regularizer="l2",
)(x)
model = tf.keras.Model(inputs, outputs, name="MyEfficient-V2B0")
model = compile_model(model)
return model
def compile_model(model):
# Compile
optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)
metrics = [
tf.keras.metrics.BinaryAccuracy(),
tf.keras.metrics.Precision(name="precision"),
tf.keras.metrics.Recall(name="recall"),
]
model.compile(
optimizer=optimizer,
loss="binary_crossentropy",
metrics=metrics,
)
return model
Keras tuner search summary¶
In [22]:
tuner = keras_tuner.RandomSearch(
hypermodel=build_model,
objective="val_binary_accuracy",
max_trials=10,
executions_per_trial=2,
seed=seed,
overwrite=True,
directory=HP_DIR,
project_name="all_multi_tuner",
)
tuner.search_space_summary()
Search space summary
Default search space size: 4
pooling (Choice)
{'default': 'average', 'conditions': [], 'values': ['average', 'global'], 'ordered': False}
norm (Boolean)
{'default': False, 'conditions': []}
units (Int)
{'default': None, 'conditions': [], 'min_value': 64, 'max_value': 512, 'step': 32, 'sampling': 'linear'}
dropout (Float)
{'default': 0.1, 'conditions': [], 'min_value': 0.1, 'max_value': 0.4, 'step': 0.1, 'sampling': 'linear'}
In [23]:
model = build_model(keras_tuner.HyperParameters())
#model = compile_model(model) # this will be called in the build_model
epochs = 1000
In [24]:
# This will be called by the tuner search
tuner.search(ds_train, epochs=epochs, validation_data=ds_val, callbacks=callbacks)
Trial 10 Complete [01h 37m 07s] val_binary_accuracy: 0.700520783662796 Best val_binary_accuracy So Far: 0.7036457657814026 Total elapsed time: 18h 25m 42s
In [25]:
# The model weights (that are considered the best) can be loaded as -
#model.load_weights(checkpoint_filepath)
model_list = tuner.get_best_models(num_models=1)
model = model_list[0] # the best model from the tuner
# print a summary of the best trials
tuner.results_summary(num_trials=1)
Results summary Results in hp_search\all_multi_tuner\all_multi_tuner Showing 1 best trials Objective(name="val_binary_accuracy", direction="max") Trial 07 summary Hyperparameters: pooling: average norm: True units: 64 dropout: 0.4 Score: 0.7036457657814026
In [26]:
# Save the best model
best_model = model_list[0] # the best model from the tuner
In [27]:
# print a summary of a few moretrials
tuner.results_summary(num_trials=5)
Results summary Results in hp_search\all_multi_tuner\all_multi_tuner Showing 5 best trials Objective(name="val_binary_accuracy", direction="max") Trial 07 summary Hyperparameters: pooling: average norm: True units: 64 dropout: 0.4 Score: 0.7036457657814026 Trial 09 summary Hyperparameters: pooling: global norm: True units: 192 dropout: 0.2 Score: 0.700520783662796 Trial 01 summary Hyperparameters: pooling: average norm: False units: 256 dropout: 0.1 Score: 0.6979166269302368 Trial 00 summary Hyperparameters: pooling: global norm: True units: 416 dropout: 0.2 Score: 0.6963541209697723 Trial 06 summary Hyperparameters: pooling: average norm: True units: 480 dropout: 0.30000000000000004 Score: 0.6963541209697723
In [28]:
# Reference: https://keras.io/api/applications/
# Layers in the base models and models
print(f"Number of layers in base model {base_model.name}: {len(base_model.layers)}")
print(f"Number of layers in feature extraction model {best_model.name}: {len(best_model.layers)}")
Number of layers in base model efficientnetv2-b0: 270 Number of layers in feature extraction model MyEfficient-V2B0: 275
Further layer unfreezing is not workable on my device.
In [26]:
# we chose to train the top 1 we will freeze
# the first 249 layers and unfreeze the rest:
#
"""
for layer in model.layers[:268]:
layer.trainable = False
for layer in model.layers[268:]:
layer.trainable = True
"""
Out[26]:
'\nfor layer in model.layers[:268]:\n layer.trainable = False\nfor layer in model.layers[268:]:\n layer.trainable = True\n'
In [27]:
"""
model = compile_model(model)
model.fit(ds_train,
epochs=epochs,
validation_data=ds_val,
callbacks=callbacks)
"""
Out[27]:
'\nmodel = compile_model(model)\nmodel.fit(ds_train,\n epochs=epochs, \n validation_data=ds_val,\n callbacks=callbacks)\n'
Save the model¶
In [29]:
# Save in tf SavedModel format
model_filepath = MODEL_TF_DIR
tf.saved_model.save(best_model, model_filepath)
WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 91). These functions will not be directly callable after loading.
INFO:tensorflow:Assets written to: model\all_multi_tuner\assets
INFO:tensorflow:Assets written to: model\all_multi_tuner\assets
In [30]:
# save in keras format
model_filepath = MODEL_KERAS_DIR
tf.keras.models.save_model(best_model, model_filepath, overwrite=True)
What would happened if the best hyperparameters are used to retrained the model?¶
Retrained the model with the best hyperparameters. However the val_binary_accuracy was not the same as the one from the best model. The reason could be that one epoch is not enough. The retraining may need many epochs to reach the val_binary_accuracy like the best model.
In [31]:
# Get the top hyperparameters.
best_hps = tuner.get_best_hyperparameters(1)
# Build the model with the best hp.
model = build_model(best_hps[0])
# Fit with the entire dataset.
# Only need to train 1 time with the best hyperparameters.
history = model.fit(ds_train, epochs=1, validation_data=ds_val, callbacks=callbacks)
114/114 [==============================] - 22s 147ms/step - loss: 1.9913 - binary_accuracy: 0.5382 - precision: 0.3684 - recall: 0.4266 - val_loss: 1.3897 - val_binary_accuracy: 0.6594 - val_precision: 0.3750 - val_recall: 0.0473
In [32]:
history.history["val_binary_accuracy"][0]
Out[32]:
0.6593749523162842
In [33]:
# Quick summary of the best metrics according to binary_accuracy
val = np.argmax(np.array(history.history["val_binary_accuracy"]))
print(f"Binary accuracy: {history.history['val_binary_accuracy'][val]}")
print(f"Precision: {history.history['val_precision'][val]}")
print(f"Recall: {history.history['val_recall'][val]}")
Binary accuracy: 0.6593749523162842 Precision: 0.375 Recall: 0.04731861129403114
In [32]:
plot_history(history)
In [ ]:
In [ ]: