ShkShahid's picture
Upload 3 files
41c6291
Raw History Blame Contribute Delete
10.4 kB
import tensorflow as tf
import tensorlayer as tl
import numpy as np
# The HDR reconstruction autoencoder fully convolutional neural network
def model(x, batch_size=1, is_training=False):
# Encoder network (VGG16, until pool5)
x_in = tf.scalar_mul(255.0, x)
net_in = tl.layers.InputLayer(x_in, name='input_layer')
conv_layers, skip_layers = encoder(net_in)
# Fully convolutional layers on top of VGG16 conv layers
network = tl.layers.Conv2dLayer(conv_layers,
act = tf.identity,
shape = [3, 3, 512, 512],
strides = [1, 1, 1, 1],
padding='SAME',
name ='encoder/h6/conv')
network = tl.layers.BatchNormLayer(network, is_train=is_training, name='encoder/h6/batch_norm')
network.outputs = tf.nn.relu(network.outputs, name='encoder/h6/relu')
# Decoder network
network = decoder(network, skip_layers, batch_size, is_training)
if is_training:
return network, conv_layers
return network
# Final prediction of the model, including blending with input
def get_final(network, x_in):
sb, sy, sx, sf = x_in.get_shape().as_list()
y_predict = network.outputs
# Highlight mask
thr = 0.05
alpha = tf.reduce_max(x_in, reduction_indices=[3])
alpha = tf.minimum(1.0, tf.maximum(0.0, alpha-1.0+thr)/thr)
alpha = tf.reshape(alpha, [-1, sy, sx, 1])
alpha = tf.tile(alpha, [1,1,1,3])
# Linearied input and prediction
x_lin = tf.pow(x_in, 2.0)
y_predict = tf.exp(y_predict)-1.0/255.0
# Alpha blending
y_final = (1-alpha)*x_lin + alpha*y_predict
return y_final
# Convolutional layers of the VGG16 model used as encoder network
def encoder(input_layer):
VGG_MEAN = [103.939, 116.779, 123.68]
# Convert RGB to BGR
red, green, blue = tf.split(input_layer.outputs, 3, 3)
bgr = tf.concat([ blue - VGG_MEAN[0], green - VGG_MEAN[1], red - VGG_MEAN[2] ], axis=3)
network = tl.layers.InputLayer(bgr, name='encoder/input_layer_bgr')
# Convolutional layers size 1
network = conv_layer(network, [ 3, 64], 'encoder/h1/conv_1')
beforepool1 = conv_layer(network, [64, 64], 'encoder/h1/conv_2')
network = pool_layer(beforepool1, 'encoder/h1/pool')
# Convolutional layers size 2
network = conv_layer(network, [64, 128], 'encoder/h2/conv_1')
beforepool2 = conv_layer(network, [128, 128], 'encoder/h2/conv_2')
network = pool_layer(beforepool2, 'encoder/h2/pool')
# Convolutional layers size 3
network = conv_layer(network, [128, 256], 'encoder/h3/conv_1')
network = conv_layer(network, [256, 256], 'encoder/h3/conv_2')
beforepool3 = conv_layer(network, [256, 256], 'encoder/h3/conv_3')
network = pool_layer(beforepool3, 'encoder/h3/pool')
# Convolutional layers size 4
network = conv_layer(network, [256, 512], 'encoder/h4/conv_1')
network = conv_layer(network, [512, 512], 'encoder/h4/conv_2')
beforepool4 = conv_layer(network, [512, 512], 'encoder/h4/conv_3')
network = pool_layer(beforepool4, 'encoder/h4/pool')
# Convolutional layers size 5
network = conv_layer(network, [512, 512], 'encoder/h5/conv_1')
network = conv_layer(network, [512, 512], 'encoder/h5/conv_2')
beforepool5 = conv_layer(network, [512, 512], 'encoder/h5/conv_3')
network = pool_layer(beforepool5, 'encoder/h5/pool')
return network, (input_layer, beforepool1, beforepool2, beforepool3, beforepool4, beforepool5)
# Decoder network
def decoder(input_layer, skip_layers, batch_size=1, is_training=False):
sb, sx, sy, sf = input_layer.outputs.get_shape().as_list()
alpha = 0.0
# Upsampling 1
network = deconv_layer(input_layer, (batch_size,sx,sy,sf,sf), 'decoder/h1/decon2d', alpha, is_training)
# Upsampling 2
network = skip_connection_layer(network, skip_layers[5], 'decoder/h2/fuse_skip_connection', is_training)
network = deconv_layer(network, (batch_size,2*sx,2*sy,sf,sf), 'decoder/h2/decon2d', alpha, is_training)
# Upsampling 3
network = skip_connection_layer(network, skip_layers[4], 'decoder/h3/fuse_skip_connection', is_training)
network = deconv_layer(network, (batch_size,4*sx,4*sy,sf,sf/2), 'decoder/h3/decon2d', alpha, is_training)
# Upsampling 4
network = skip_connection_layer(network, skip_layers[3], 'decoder/h4/fuse_skip_connection', is_training)
network = deconv_layer(network, (batch_size,8*sx,8*sy,sf/2,sf/4), 'decoder/h4/decon2d', alpha, is_training)
# Upsampling 5
network = skip_connection_layer(network, skip_layers[2], 'decoder/h5/fuse_skip_connection', is_training)
network = deconv_layer(network, (batch_size,16*sx,16*sy,sf/4,sf/8), 'decoder/h5/decon2d', alpha, is_training)
# Skip-connection at full size
network = skip_connection_layer(network, skip_layers[1], 'decoder/h6/fuse_skip_connection', is_training)
# Final convolution
network = tl.layers.Conv2dLayer(network,
act = tf.identity,
shape = [1, 1, int(sf/8), 3],
strides=[1, 1, 1, 1],
padding='SAME',
W_init = tf.contrib.layers.xavier_initializer(uniform=False),
b_init = tf.constant_initializer(value=0.0),
name ='decoder/h7/conv2d')
# Final skip-connection
network = tl.layers.BatchNormLayer(network, is_train=is_training, name='decoder/h7/batch_norm')
network.outputs = tf.maximum(alpha*network.outputs, network.outputs, name='decoder/h7/leaky_relu')
network = skip_connection_layer(network, skip_layers[0], 'decoder/h7/fuse_skip_connection')
return network
# Load weights for VGG16 encoder convolutional layers
# Weights are from a .npy file generated with the caffe-tensorflow tool
def load_vgg_weights(network, weight_file, session):
params = []
if weight_file.lower().endswith('.npy'):
npy = np.load(weight_file, encoding='latin1')
for key, val in sorted(npy.item().items()):
if(key[:4] == "conv"):
print(" Loading %s" % (key))
print(" weights with size %s " % str(val['weights'].shape))
print(" and biases with size %s " % str(val['biases'].shape))
params.append(val['weights'])
params.append(val['biases'])
else:
print('No weights in suitable .npy format found for path ', weight_file)
print('Assigning loaded weights..')
tl.files.assign_params(session, params, network)
return network
# === Layers ==================================================================
# Convolutional layer
def conv_layer(input_layer, sz, str):
network = tl.layers.Conv2dLayer(input_layer,
act = tf.nn.relu,
shape = [3, 3, sz[0], sz[1]],
strides = [1, 1, 1, 1],
padding = 'SAME',
name = str)
return network
# Max-pooling layer
def pool_layer(input_layer, str):
network = tl.layers.PoolLayer(input_layer,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
pool = tf.nn.max_pool,
name = str)
return network
# Concatenating fusion of skip-connections
def skip_connection_layer(input_layer, skip_layer, str, is_training=False):
_, sx, sy, sf = input_layer.outputs.get_shape().as_list()
_, sx_, sy_, sf_ = skip_layer.outputs.get_shape().as_list()
assert (sx_,sy_,sf_) == (sx,sy,sf)
# skip-connection domain transformation, from LDR encoder to log HDR decoder
skip_layer.outputs = tf.log(tf.pow(tf.scalar_mul(1.0/255, skip_layer.outputs), 2.0)+1.0/255.0)
# specify weights for fusion of concatenation, so that it performs an element-wise addition
weights = np.zeros((1, 1, sf+sf_, sf))
for i in range(sf):
weights[0, 0, i, i] = 1
weights[:, :, i+sf_, i] = 1
add_init = tf.constant_initializer(value=weights, dtype=tf.float32)
# concatenate layers
network = tl.layers.ConcatLayer([input_layer,skip_layer], concat_dim=3, name ='%s/skip_connection'%str)
# fuse concatenated layers using the specified weights for initialization
network = tl.layers.Conv2dLayer(network,
act = tf.identity,
shape = [1, 1, sf+sf_, sf],
strides = [1, 1, 1, 1],
padding = 'SAME',
W_init = add_init,
b_init = tf.constant_initializer(value=0.0),
name = str)
return network
# Deconvolution layer
def deconv_layer(input_layer, sz, str, alpha, is_training=False):
scale = 2
filter_size = (2 * scale - scale % 2)
num_in_channels = int(sz[3])
num_out_channels = int(sz[4])
# create bilinear weights in numpy array
bilinear_kernel = np.zeros([filter_size, filter_size], dtype=np.float32)
scale_factor = (filter_size + 1) // 2
if filter_size % 2 == 1:
center = scale_factor - 1
else:
center = scale_factor - 0.5
for x in range(filter_size):
for y in range(filter_size):
bilinear_kernel[x,y] = (1 - abs(x - center) / scale_factor) * \
(1 - abs(y - center) / scale_factor)
weights = np.zeros((filter_size, filter_size, num_out_channels, num_in_channels))
for i in range(num_out_channels):
weights[:, :, i, i] = bilinear_kernel
init_matrix = tf.constant_initializer(value=weights, dtype=tf.float32)
network = tl.layers.DeConv2dLayer(input_layer,
shape = [filter_size, filter_size, num_out_channels, num_in_channels],
output_shape = [sz[0], sz[1]*scale, sz[2]*scale, num_out_channels],
strides=[1, scale, scale, 1],
W_init=init_matrix,
padding='SAME',
act=tf.identity,
name=str)
network = tl.layers.BatchNormLayer(network, is_train=is_training, name='%s/batch_norm_dc'%str)
network.outputs = tf.maximum(alpha*network.outputs, network.outputs, name='%s/leaky_relu_dc'%str)
return network