Li-zhangsheng's picture
Update CD-DCU-Net(Incorporate CD-module)/CD-DCU-Net.py
cf4da0c verified
Raw History Blame Contribute Delete
10.7 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchsummary import summary
import numpy as np
class ConcreteDropout(nn.Module):
def __init__(self, weight_regularizer=1e-7, dropout_regularizer=1e-6, init_min=0.1, init_max=0.4):
super().__init__()
self.weight_regularizer = weight_regularizer
self.dropout_regularizer = dropout_regularizer
init_min = np.log(init_min) - np.log(1. - init_min)
init_max = np.log(init_max) - np.log(1. - init_max)
self.p_logit = nn.Parameter(torch.empty(1).uniform_(init_min, init_max))
self.regularization_loss = 0.0
def forward(self, x, layer):
p = torch.sigmoid(self.p_logit)
p = torch.clamp(p, max=0.4)
if self.training:
# Training mode: use Concrete Dropout.
out = layer(self._concrete_dropout(x, p))
# Calculate the regularization loss.
sum_of_square = 0
for param in layer.parameters():
sum_of_square += torch.sum(torch.pow(param, 2))
weights_regularizer = self.weight_regularizer * sum_of_square / (1 - p)
dropout_regularizer = p * torch.log(p)
dropout_regularizer += (1. - p) * torch.log(1. - p)
input_dim = x[0].numel()
#input_dim=x.size(1)
dropout_regularizer *= self.dropout_regularizer * input_dim
self.regularization_loss = (weights_regularizer + dropout_regularizer)
else:
# Evaluation mode: pass directly through the layer.
out = layer(x)
return out
def _concrete_dropout(self, x, p):
eps = 1e-7
temp = 0.5
unif_noise = torch.rand_like(x, device=x.device)
drop_prob = (torch.log(p + eps)
- torch.log(1 - p + eps)
+ torch.log(unif_noise + eps)
- torch.log(1 - unif_noise + eps))
drop_prob = torch.sigmoid(drop_prob / temp)
random_tensor = 1 - drop_prob
retain_prob = 1 - p
x = torch.mul(x, random_tensor)
x /= retain_prob
return x
def get_dropout_rate(self):
"""Return the current dropout rate."""
return torch.sigmoid(self.p_logit).item()
class DoubleConv(nn.Module):
def __init__(self, in_channels, out_channels, mid_channels=None):
super().__init__()
if not mid_channels:
mid_channels = out_channels
self.double_conv = nn.Sequential(
nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(mid_channels),
nn.ReLU(inplace=True),
nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1, bias=False),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True),
)
def forward(self, x):
return self.double_conv(x)
# Downsample with max pooling.
class Down(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.maxpool_conv = nn.Sequential(
nn.MaxPool2d(2),
DoubleConv(in_channels, out_channels,)
)
def forward(self, x):
return self.maxpool_conv(x)
class Up(nn.Module):
def __init__(self, in_channels, out_channels, bilinear=False):
super().__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
self.conv = DoubleConv(in_channels + in_channels // 2, out_channels)
else:
self.up = nn.ConvTranspose2d(in_channels, in_channels // 2, kernel_size=2, stride=2)
self.conv = DoubleConv(in_channels // 2 + out_channels, out_channels)
def forward(self, x1, x2):
x1 = self.up(x1)
diffY = x2.size()[2] - x1.size()[2]
diffX = x2.size()[3] - x1.size()[3]
x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2,
diffY // 2, diffY - diffY // 2])
x = torch.cat([x2, x1], dim=1)
return self.conv(x)
class OutConv(nn.Module):
def __init__(self, in_channels, out_channels=4):
super(OutConv, self).__init__()
self.conv = nn.Sequential(
nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1, bias=True),
nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, bias=True)
)
def forward(self, x):
return self.conv(x)
class UNet(nn.Module):
def __init__(self, n_channels=3, bilinear=False):
super(UNet, self).__init__()
self.n_channels = n_channels
self.bilinear = bilinear
# Encoder.
self.inc = DoubleConv(n_channels, 64)
self.down1 = Down(64, 128)
self.down2 = Down(128, 256)
self.down3 = Down(256, 512)
self.down4 = Down(512, 1024)
# Encoder Concrete Dropout layers: apply one at every level.
self.cd_enc1 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_enc2 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_enc3 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_enc4 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_enc5 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
# Convolution layers paired with encoder Concrete Dropout.
self.cd_enc_conv1 = nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False)
self.cd_enc_conv2 = nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False)
self.cd_enc_conv3 = nn.Conv2d(256, 256, kernel_size=3, padding=1, bias=False)
self.cd_enc_conv4 = nn.Conv2d(512, 512, kernel_size=3, padding=1, bias=False)
self.cd_enc_conv5 = nn.Conv2d(1024, 1024, kernel_size=3, padding=1, bias=False)
# Decoder.
self.up1 = Up(1024, 512, bilinear)
self.up2 = Up(512, 256, bilinear)
self.up3 = Up(256, 128, bilinear)
self.up4 = Up(128, 64, bilinear)
# Decoder Concrete Dropout layers.
self.cd_dec1 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_dec2 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_dec3 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
self.cd_dec4 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5)
# Convolution layers paired with decoder Concrete Dropout.
self.cd_dec_conv1 = nn.Conv2d(512, 512, kernel_size=3, padding=1, bias=False)
self.cd_dec_conv2 = nn.Conv2d(256, 256, kernel_size=3, padding=1, bias=False)
self.cd_dec_conv3 = nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False)
self.cd_dec_conv4 = nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False)
self.outc = OutConv(64, 4)
def forward(self, x):
# Encoder path.
x1 = self.inc(x)
x1 = self.cd_enc1(x1, self.cd_enc_conv1) # Apply Concrete Dropout at level 1.
x2 = self.down1(x1)
x2 = self.cd_enc2(x2, self.cd_enc_conv2) # Apply Concrete Dropout at level 2.
x3 = self.down2(x2)
x3 = self.cd_enc3(x3, self.cd_enc_conv3) # Apply Concrete Dropout at level 3.
x4 = self.down3(x3)
x4 = self.cd_enc4(x4, self.cd_enc_conv4) # Apply Concrete Dropout at level 4.
x5 = self.down4(x4)
x5 = self.cd_enc5(x5, self.cd_enc_conv5) # Apply Concrete Dropout at the bottleneck.
# Decoder path.
x = self.up1(x5, x4)
x = self.cd_dec1(x, self.cd_dec_conv1) # Apply Concrete Dropout at decoder level 1.
x = self.up2(x, x3)
x = self.cd_dec2(x, self.cd_dec_conv2) # Apply Concrete Dropout at decoder level 2.
x = self.up3(x, x2)
x = self.cd_dec3(x, self.cd_dec_conv3) # Apply Concrete Dropout at decoder level 3.
x = self.up4(x, x1)
x = self.cd_dec4(x, self.cd_dec_conv4) # Apply Concrete Dropout at decoder level 4.
# Predict means and variances.
outputs = self.outc(x)
M_mean = outputs[:, 0:1] # M mean.
M_var = 0.001 + F.softplus(outputs[:, 2:3]) # M variance; softplus keeps it positive.
# M_var = F.softplus(outputs[:, 2:3]) # M variance; softplus keeps it positive.
D_mean = outputs[:, 1:2] # D mean.
D_var = 0.001 + F.softplus(outputs[:, 3:4]) # D variance; softplus keeps it positive.
# D_var = F.softplus(outputs[:, 3:4])
return {'M_mean': M_mean, 'M_var': M_var, 'D_mean': D_mean, 'D_var': D_var}
def get_regularization_loss(self):
"""Return the regularization loss from all Concrete Dropout layers."""
return (self.cd_enc1.regularization_loss +
self.cd_enc2.regularization_loss +
self.cd_enc3.regularization_loss +
self.cd_enc4.regularization_loss +
self.cd_enc5.regularization_loss +
self.cd_dec1.regularization_loss +
self.cd_dec2.regularization_loss +
self.cd_dec3.regularization_loss +
self.cd_dec4.regularization_loss)
def get_dropout_rates(self):
"""Return the dropout rate for every layer."""
return {
'enc1': self.cd_enc1.get_dropout_rate(),
'enc2': self.cd_enc2.get_dropout_rate(),
'enc3': self.cd_enc3.get_dropout_rate(),
'enc4': self.cd_enc4.get_dropout_rate(),
'enc5': self.cd_enc5.get_dropout_rate(),
'dec1': self.cd_dec1.get_dropout_rate(),
'dec2': self.cd_dec2.get_dropout_rate(),
'dec3': self.cd_dec3.get_dropout_rate(),
'dec4': self.cd_dec4.get_dropout_rate()
}
if __name__ == "__main__":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = UNet(3, bilinear=False).to(device)
print(summary(model, (3, 432, 352)))
# Example input.
x = torch.randn(1, 3, 800, 800).to(device)
output = model(x)
print("Output mean M:", output['M_mean'].shape)
print("Output variance M:", output['M_var'].shape)
print("Output mean D:", output['D_mean'].shape)
print("Output variance D:", output['D_var'].shape)
# Print the dropout rate for every layer.
dropout_rates = model.get_dropout_rates()
for layer, rate in dropout_rates.items():
print(f"{layer} dropout rate: {rate:.4f}")