ShkShahid commited on
Commit
41c6291
·
1 Parent(s): b12998b

Upload 3 files

Browse files
Files changed (3) hide show
  1. hdrcnn_params.npz +3 -0
  2. img_io.py +88 -0
  3. network.py +263 -0
hdrcnn_params.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fda5e56ac431ab4db2310c61d995d2ed3b4defc4d8d48c5b0c195782814edaf1
3
+ size 177212377
img_io.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+
4
+ import numpy as np
5
+ import scipy.misc
6
+ #import OpenEXR, Imath
7
+
8
+ class IOException(Exception):
9
+ def __init__(self, value):
10
+ self.value = value
11
+ def __str__(self):
12
+ return repr(self.value)
13
+
14
+ # Read and prepare 8-bit image in a specified resolution
15
+ def readLDR(file, sz, clip=True, sc=1.0):
16
+ try:
17
+ x_buffer = scipy.misc.imread(file)
18
+
19
+ # Clip image, so that ratio is not changed by image resize
20
+ if clip:
21
+ sz_in = [float(x) for x in x_buffer.shape]
22
+ sz_out = [float(x) for x in sz]
23
+
24
+ r_in = sz_in[1]/sz_in[0]
25
+ r_out = sz_out[1]/sz_out[0]
26
+
27
+ if r_out / r_in > 1.0:
28
+ sx = sz_in[1]
29
+ sy = sx/r_out
30
+ else:
31
+ sy = sz_in[0]
32
+ sx = sy*r_out
33
+
34
+ yo = np.maximum(0.0, (sz_in[0]-sy)/2.0)
35
+ xo = np.maximum(0.0, (sz_in[1]-sx)/2.0)
36
+
37
+ x_buffer = x_buffer[int(yo):int(yo+sy),int(xo):int(xo+sx),:]
38
+
39
+ # Image resize and conversion to float
40
+ x_buffer = scipy.misc.imresize(x_buffer, sz)
41
+ x_buffer = x_buffer.astype(np.float32)/255.0
42
+
43
+ # Scaling and clipping
44
+ if sc > 1.0:
45
+ x_buffer = np.minimum(1.0, sc*x_buffer)
46
+
47
+ x_buffer = x_buffer[np.newaxis,:,:,:]
48
+
49
+ return x_buffer
50
+
51
+ except Exception as e:
52
+ raise IOException("Failed reading LDR image: %s"%e)
53
+
54
+ # Write exposure compensated 8-bit image
55
+ def writeLDR(img, file, exposure=0):
56
+
57
+ # Convert exposure fstop in linear domain to scaling factor on display values
58
+ sc = np.power(np.power(2.0, exposure), 0.5)
59
+
60
+ try:
61
+ scipy.misc.toimage(sc*np.squeeze(img), cmin=0.0, cmax=1.0).save(file)
62
+ except Exception as e:
63
+ raise IOException("Failed writing LDR image: %s"%e)
64
+
65
+ # Write HDR image using OpenEXR
66
+
67
+
68
+
69
+ # Read training data (HDR ground truth and LDR JPEG images)
70
+ def load_training_pair(name_hdr, name_jpg):
71
+
72
+ data = np.fromfile(name_hdr, dtype=np.float32)
73
+ ss = len(data)
74
+
75
+ if ss < 3:
76
+ return (False,0,0)
77
+
78
+ sz = np.floor(data[0:3]).astype(int)
79
+ npix = sz[0]*sz[1]*sz[2]
80
+ meta_length = ss - npix
81
+
82
+ # Read binary HDR ground truth
83
+ y = np.reshape(data[meta_length:meta_length+npix], (sz[0], sz[1], sz[2]))
84
+
85
+ # Read JPEG LDR image
86
+ x = scipy.misc.imread(name_jpg).astype(np.float32)/255.0
87
+
88
+ return (True,x,y)
network.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+
4
+ import tensorflow as tf
5
+ import tensorlayer as tl
6
+ import numpy as np
7
+
8
+ # The HDR reconstruction autoencoder fully convolutional neural network
9
+ def model(x, batch_size=1, is_training=False):
10
+
11
+ # Encoder network (VGG16, until pool5)
12
+ x_in = tf.scalar_mul(255.0, x)
13
+ net_in = tl.layers.InputLayer(x_in, name='input_layer')
14
+ conv_layers, skip_layers = encoder(net_in)
15
+
16
+ # Fully convolutional layers on top of VGG16 conv layers
17
+ network = tl.layers.Conv2dLayer(conv_layers,
18
+ act = tf.identity,
19
+ shape = [3, 3, 512, 512],
20
+ strides = [1, 1, 1, 1],
21
+ padding='SAME',
22
+ name ='encoder/h6/conv')
23
+ network = tl.layers.BatchNormLayer(network, is_train=is_training, name='encoder/h6/batch_norm')
24
+ network.outputs = tf.nn.relu(network.outputs, name='encoder/h6/relu')
25
+
26
+ # Decoder network
27
+ network = decoder(network, skip_layers, batch_size, is_training)
28
+
29
+ if is_training:
30
+ return network, conv_layers
31
+
32
+ return network
33
+
34
+
35
+ # Final prediction of the model, including blending with input
36
+ def get_final(network, x_in):
37
+ sb, sy, sx, sf = x_in.get_shape().as_list()
38
+ y_predict = network.outputs
39
+
40
+ # Highlight mask
41
+ thr = 0.05
42
+ alpha = tf.reduce_max(x_in, reduction_indices=[3])
43
+ alpha = tf.minimum(1.0, tf.maximum(0.0, alpha-1.0+thr)/thr)
44
+ alpha = tf.reshape(alpha, [-1, sy, sx, 1])
45
+ alpha = tf.tile(alpha, [1,1,1,3])
46
+
47
+ # Linearied input and prediction
48
+ x_lin = tf.pow(x_in, 2.0)
49
+ y_predict = tf.exp(y_predict)-1.0/255.0
50
+
51
+ # Alpha blending
52
+ y_final = (1-alpha)*x_lin + alpha*y_predict
53
+
54
+ return y_final
55
+
56
+
57
+ # Convolutional layers of the VGG16 model used as encoder network
58
+ def encoder(input_layer):
59
+
60
+ VGG_MEAN = [103.939, 116.779, 123.68]
61
+
62
+ # Convert RGB to BGR
63
+ red, green, blue = tf.split(input_layer.outputs, 3, 3)
64
+ bgr = tf.concat([ blue - VGG_MEAN[0], green - VGG_MEAN[1], red - VGG_MEAN[2] ], axis=3)
65
+
66
+ network = tl.layers.InputLayer(bgr, name='encoder/input_layer_bgr')
67
+
68
+ # Convolutional layers size 1
69
+ network = conv_layer(network, [ 3, 64], 'encoder/h1/conv_1')
70
+ beforepool1 = conv_layer(network, [64, 64], 'encoder/h1/conv_2')
71
+ network = pool_layer(beforepool1, 'encoder/h1/pool')
72
+
73
+ # Convolutional layers size 2
74
+ network = conv_layer(network, [64, 128], 'encoder/h2/conv_1')
75
+ beforepool2 = conv_layer(network, [128, 128], 'encoder/h2/conv_2')
76
+ network = pool_layer(beforepool2, 'encoder/h2/pool')
77
+
78
+ # Convolutional layers size 3
79
+ network = conv_layer(network, [128, 256], 'encoder/h3/conv_1')
80
+ network = conv_layer(network, [256, 256], 'encoder/h3/conv_2')
81
+ beforepool3 = conv_layer(network, [256, 256], 'encoder/h3/conv_3')
82
+ network = pool_layer(beforepool3, 'encoder/h3/pool')
83
+
84
+ # Convolutional layers size 4
85
+ network = conv_layer(network, [256, 512], 'encoder/h4/conv_1')
86
+ network = conv_layer(network, [512, 512], 'encoder/h4/conv_2')
87
+ beforepool4 = conv_layer(network, [512, 512], 'encoder/h4/conv_3')
88
+ network = pool_layer(beforepool4, 'encoder/h4/pool')
89
+
90
+ # Convolutional layers size 5
91
+ network = conv_layer(network, [512, 512], 'encoder/h5/conv_1')
92
+ network = conv_layer(network, [512, 512], 'encoder/h5/conv_2')
93
+ beforepool5 = conv_layer(network, [512, 512], 'encoder/h5/conv_3')
94
+ network = pool_layer(beforepool5, 'encoder/h5/pool')
95
+
96
+ return network, (input_layer, beforepool1, beforepool2, beforepool3, beforepool4, beforepool5)
97
+
98
+
99
+ # Decoder network
100
+ def decoder(input_layer, skip_layers, batch_size=1, is_training=False):
101
+ sb, sx, sy, sf = input_layer.outputs.get_shape().as_list()
102
+ alpha = 0.0
103
+
104
+ # Upsampling 1
105
+ network = deconv_layer(input_layer, (batch_size,sx,sy,sf,sf), 'decoder/h1/decon2d', alpha, is_training)
106
+
107
+ # Upsampling 2
108
+ network = skip_connection_layer(network, skip_layers[5], 'decoder/h2/fuse_skip_connection', is_training)
109
+ network = deconv_layer(network, (batch_size,2*sx,2*sy,sf,sf), 'decoder/h2/decon2d', alpha, is_training)
110
+
111
+ # Upsampling 3
112
+ network = skip_connection_layer(network, skip_layers[4], 'decoder/h3/fuse_skip_connection', is_training)
113
+ network = deconv_layer(network, (batch_size,4*sx,4*sy,sf,sf/2), 'decoder/h3/decon2d', alpha, is_training)
114
+
115
+ # Upsampling 4
116
+ network = skip_connection_layer(network, skip_layers[3], 'decoder/h4/fuse_skip_connection', is_training)
117
+ network = deconv_layer(network, (batch_size,8*sx,8*sy,sf/2,sf/4), 'decoder/h4/decon2d', alpha, is_training)
118
+
119
+ # Upsampling 5
120
+ network = skip_connection_layer(network, skip_layers[2], 'decoder/h5/fuse_skip_connection', is_training)
121
+ network = deconv_layer(network, (batch_size,16*sx,16*sy,sf/4,sf/8), 'decoder/h5/decon2d', alpha, is_training)
122
+
123
+ # Skip-connection at full size
124
+ network = skip_connection_layer(network, skip_layers[1], 'decoder/h6/fuse_skip_connection', is_training)
125
+
126
+ # Final convolution
127
+ network = tl.layers.Conv2dLayer(network,
128
+ act = tf.identity,
129
+ shape = [1, 1, int(sf/8), 3],
130
+ strides=[1, 1, 1, 1],
131
+ padding='SAME',
132
+ W_init = tf.contrib.layers.xavier_initializer(uniform=False),
133
+ b_init = tf.constant_initializer(value=0.0),
134
+ name ='decoder/h7/conv2d')
135
+
136
+ # Final skip-connection
137
+ network = tl.layers.BatchNormLayer(network, is_train=is_training, name='decoder/h7/batch_norm')
138
+ network.outputs = tf.maximum(alpha*network.outputs, network.outputs, name='decoder/h7/leaky_relu')
139
+ network = skip_connection_layer(network, skip_layers[0], 'decoder/h7/fuse_skip_connection')
140
+
141
+ return network
142
+
143
+
144
+ # Load weights for VGG16 encoder convolutional layers
145
+ # Weights are from a .npy file generated with the caffe-tensorflow tool
146
+ def load_vgg_weights(network, weight_file, session):
147
+ params = []
148
+
149
+ if weight_file.lower().endswith('.npy'):
150
+ npy = np.load(weight_file, encoding='latin1')
151
+ for key, val in sorted(npy.item().items()):
152
+ if(key[:4] == "conv"):
153
+ print(" Loading %s" % (key))
154
+ print(" weights with size %s " % str(val['weights'].shape))
155
+ print(" and biases with size %s " % str(val['biases'].shape))
156
+ params.append(val['weights'])
157
+ params.append(val['biases'])
158
+ else:
159
+ print('No weights in suitable .npy format found for path ', weight_file)
160
+
161
+ print('Assigning loaded weights..')
162
+ tl.files.assign_params(session, params, network)
163
+
164
+ return network
165
+
166
+
167
+ # === Layers ==================================================================
168
+
169
+ # Convolutional layer
170
+ def conv_layer(input_layer, sz, str):
171
+ network = tl.layers.Conv2dLayer(input_layer,
172
+ act = tf.nn.relu,
173
+ shape = [3, 3, sz[0], sz[1]],
174
+ strides = [1, 1, 1, 1],
175
+ padding = 'SAME',
176
+ name = str)
177
+
178
+ return network
179
+
180
+
181
+ # Max-pooling layer
182
+ def pool_layer(input_layer, str):
183
+ network = tl.layers.PoolLayer(input_layer,
184
+ ksize=[1, 2, 2, 1],
185
+ strides=[1, 2, 2, 1],
186
+ padding='SAME',
187
+ pool = tf.nn.max_pool,
188
+ name = str)
189
+
190
+ return network
191
+
192
+
193
+ # Concatenating fusion of skip-connections
194
+ def skip_connection_layer(input_layer, skip_layer, str, is_training=False):
195
+ _, sx, sy, sf = input_layer.outputs.get_shape().as_list()
196
+ _, sx_, sy_, sf_ = skip_layer.outputs.get_shape().as_list()
197
+
198
+ assert (sx_,sy_,sf_) == (sx,sy,sf)
199
+
200
+ # skip-connection domain transformation, from LDR encoder to log HDR decoder
201
+ skip_layer.outputs = tf.log(tf.pow(tf.scalar_mul(1.0/255, skip_layer.outputs), 2.0)+1.0/255.0)
202
+
203
+ # specify weights for fusion of concatenation, so that it performs an element-wise addition
204
+ weights = np.zeros((1, 1, sf+sf_, sf))
205
+ for i in range(sf):
206
+ weights[0, 0, i, i] = 1
207
+ weights[:, :, i+sf_, i] = 1
208
+ add_init = tf.constant_initializer(value=weights, dtype=tf.float32)
209
+
210
+ # concatenate layers
211
+ network = tl.layers.ConcatLayer([input_layer,skip_layer], concat_dim=3, name ='%s/skip_connection'%str)
212
+
213
+ # fuse concatenated layers using the specified weights for initialization
214
+ network = tl.layers.Conv2dLayer(network,
215
+ act = tf.identity,
216
+ shape = [1, 1, sf+sf_, sf],
217
+ strides = [1, 1, 1, 1],
218
+ padding = 'SAME',
219
+ W_init = add_init,
220
+ b_init = tf.constant_initializer(value=0.0),
221
+ name = str)
222
+
223
+ return network
224
+
225
+
226
+ # Deconvolution layer
227
+ def deconv_layer(input_layer, sz, str, alpha, is_training=False):
228
+ scale = 2
229
+
230
+ filter_size = (2 * scale - scale % 2)
231
+ num_in_channels = int(sz[3])
232
+ num_out_channels = int(sz[4])
233
+
234
+ # create bilinear weights in numpy array
235
+ bilinear_kernel = np.zeros([filter_size, filter_size], dtype=np.float32)
236
+ scale_factor = (filter_size + 1) // 2
237
+ if filter_size % 2 == 1:
238
+ center = scale_factor - 1
239
+ else:
240
+ center = scale_factor - 0.5
241
+ for x in range(filter_size):
242
+ for y in range(filter_size):
243
+ bilinear_kernel[x,y] = (1 - abs(x - center) / scale_factor) * \
244
+ (1 - abs(y - center) / scale_factor)
245
+ weights = np.zeros((filter_size, filter_size, num_out_channels, num_in_channels))
246
+ for i in range(num_out_channels):
247
+ weights[:, :, i, i] = bilinear_kernel
248
+
249
+ init_matrix = tf.constant_initializer(value=weights, dtype=tf.float32)
250
+
251
+ network = tl.layers.DeConv2dLayer(input_layer,
252
+ shape = [filter_size, filter_size, num_out_channels, num_in_channels],
253
+ output_shape = [sz[0], sz[1]*scale, sz[2]*scale, num_out_channels],
254
+ strides=[1, scale, scale, 1],
255
+ W_init=init_matrix,
256
+ padding='SAME',
257
+ act=tf.identity,
258
+ name=str)
259
+
260
+ network = tl.layers.BatchNormLayer(network, is_train=is_training, name='%s/batch_norm_dc'%str)
261
+ network.outputs = tf.maximum(alpha*network.outputs, network.outputs, name='%s/leaky_relu_dc'%str)
262
+
263
+ return network