import torch import torch.nn as nn import torch.nn.functional as F from torchsummary import summary import numpy as np class ConcreteDropout(nn.Module): def __init__(self, weight_regularizer=1e-7, dropout_regularizer=1e-6, init_min=0.1, init_max=0.4): super().__init__() self.weight_regularizer = weight_regularizer self.dropout_regularizer = dropout_regularizer init_min = np.log(init_min) - np.log(1. - init_min) init_max = np.log(init_max) - np.log(1. - init_max) self.p_logit = nn.Parameter(torch.empty(1).uniform_(init_min, init_max)) self.regularization_loss = 0.0 def forward(self, x, layer): p = torch.sigmoid(self.p_logit) p = torch.clamp(p, max=0.4) if self.training: # Training mode: use Concrete Dropout. out = layer(self._concrete_dropout(x, p)) # Calculate the regularization loss. sum_of_square = 0 for param in layer.parameters(): sum_of_square += torch.sum(torch.pow(param, 2)) weights_regularizer = self.weight_regularizer * sum_of_square / (1 - p) dropout_regularizer = p * torch.log(p) dropout_regularizer += (1. - p) * torch.log(1. - p) input_dim = x[0].numel() #input_dim=x.size(1) dropout_regularizer *= self.dropout_regularizer * input_dim self.regularization_loss = (weights_regularizer + dropout_regularizer) else: # Evaluation mode: pass directly through the layer. out = layer(x) return out def _concrete_dropout(self, x, p): eps = 1e-7 temp = 0.5 unif_noise = torch.rand_like(x, device=x.device) drop_prob = (torch.log(p + eps) - torch.log(1 - p + eps) + torch.log(unif_noise + eps) - torch.log(1 - unif_noise + eps)) drop_prob = torch.sigmoid(drop_prob / temp) random_tensor = 1 - drop_prob retain_prob = 1 - p x = torch.mul(x, random_tensor) x /= retain_prob return x def get_dropout_rate(self): """Return the current dropout rate.""" return torch.sigmoid(self.p_logit).item() class DoubleConv(nn.Module): def __init__(self, in_channels, out_channels, mid_channels=None): super().__init__() if not mid_channels: mid_channels = out_channels self.double_conv = nn.Sequential( nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(mid_channels), nn.ReLU(inplace=True), nn.Conv2d(mid_channels, out_channels, kernel_size=3, padding=1, bias=False), nn.BatchNorm2d(out_channels), nn.ReLU(inplace=True), ) def forward(self, x): return self.double_conv(x) # Downsample with max pooling. class Down(nn.Module): def __init__(self, in_channels, out_channels): super().__init__() self.maxpool_conv = nn.Sequential( nn.MaxPool2d(2), DoubleConv(in_channels, out_channels,) ) def forward(self, x): return self.maxpool_conv(x) class Up(nn.Module): def __init__(self, in_channels, out_channels, bilinear=False): super().__init__() if bilinear: self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True) self.conv = DoubleConv(in_channels + in_channels // 2, out_channels) else: self.up = nn.ConvTranspose2d(in_channels, in_channels // 2, kernel_size=2, stride=2) self.conv = DoubleConv(in_channels // 2 + out_channels, out_channels) def forward(self, x1, x2): x1 = self.up(x1) diffY = x2.size()[2] - x1.size()[2] diffX = x2.size()[3] - x1.size()[3] x1 = F.pad(x1, [diffX // 2, diffX - diffX // 2, diffY // 2, diffY - diffY // 2]) x = torch.cat([x2, x1], dim=1) return self.conv(x) class OutConv(nn.Module): def __init__(self, in_channels, out_channels=4): super(OutConv, self).__init__() self.conv = nn.Sequential( nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1, bias=True), nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, bias=True) ) def forward(self, x): return self.conv(x) class UNet(nn.Module): def __init__(self, n_channels=3, bilinear=False): super(UNet, self).__init__() self.n_channels = n_channels self.bilinear = bilinear # Encoder. self.inc = DoubleConv(n_channels, 64) self.down1 = Down(64, 128) self.down2 = Down(128, 256) self.down3 = Down(256, 512) self.down4 = Down(512, 1024) # Encoder Concrete Dropout layers: apply one at every level. self.cd_enc1 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_enc2 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_enc3 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_enc4 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_enc5 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) # Convolution layers paired with encoder Concrete Dropout. self.cd_enc_conv1 = nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False) self.cd_enc_conv2 = nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False) self.cd_enc_conv3 = nn.Conv2d(256, 256, kernel_size=3, padding=1, bias=False) self.cd_enc_conv4 = nn.Conv2d(512, 512, kernel_size=3, padding=1, bias=False) self.cd_enc_conv5 = nn.Conv2d(1024, 1024, kernel_size=3, padding=1, bias=False) # Decoder. self.up1 = Up(1024, 512, bilinear) self.up2 = Up(512, 256, bilinear) self.up3 = Up(256, 128, bilinear) self.up4 = Up(128, 64, bilinear) # Decoder Concrete Dropout layers. self.cd_dec1 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_dec2 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_dec3 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) self.cd_dec4 = ConcreteDropout(weight_regularizer=1e-6, dropout_regularizer=1e-5) # Convolution layers paired with decoder Concrete Dropout. self.cd_dec_conv1 = nn.Conv2d(512, 512, kernel_size=3, padding=1, bias=False) self.cd_dec_conv2 = nn.Conv2d(256, 256, kernel_size=3, padding=1, bias=False) self.cd_dec_conv3 = nn.Conv2d(128, 128, kernel_size=3, padding=1, bias=False) self.cd_dec_conv4 = nn.Conv2d(64, 64, kernel_size=3, padding=1, bias=False) self.outc = OutConv(64, 4) def forward(self, x): # Encoder path. x1 = self.inc(x) x1 = self.cd_enc1(x1, self.cd_enc_conv1) # Apply Concrete Dropout at level 1. x2 = self.down1(x1) x2 = self.cd_enc2(x2, self.cd_enc_conv2) # Apply Concrete Dropout at level 2. x3 = self.down2(x2) x3 = self.cd_enc3(x3, self.cd_enc_conv3) # Apply Concrete Dropout at level 3. x4 = self.down3(x3) x4 = self.cd_enc4(x4, self.cd_enc_conv4) # Apply Concrete Dropout at level 4. x5 = self.down4(x4) x5 = self.cd_enc5(x5, self.cd_enc_conv5) # Apply Concrete Dropout at the bottleneck. # Decoder path. x = self.up1(x5, x4) x = self.cd_dec1(x, self.cd_dec_conv1) # Apply Concrete Dropout at decoder level 1. x = self.up2(x, x3) x = self.cd_dec2(x, self.cd_dec_conv2) # Apply Concrete Dropout at decoder level 2. x = self.up3(x, x2) x = self.cd_dec3(x, self.cd_dec_conv3) # Apply Concrete Dropout at decoder level 3. x = self.up4(x, x1) x = self.cd_dec4(x, self.cd_dec_conv4) # Apply Concrete Dropout at decoder level 4. # Predict means and variances. outputs = self.outc(x) M_mean = outputs[:, 0:1] # M mean. M_var = 0.001 + F.softplus(outputs[:, 2:3]) # M variance; softplus keeps it positive. # M_var = F.softplus(outputs[:, 2:3]) # M variance; softplus keeps it positive. D_mean = outputs[:, 1:2] # D mean. D_var = 0.001 + F.softplus(outputs[:, 3:4]) # D variance; softplus keeps it positive. # D_var = F.softplus(outputs[:, 3:4]) return {'M_mean': M_mean, 'M_var': M_var, 'D_mean': D_mean, 'D_var': D_var} def get_regularization_loss(self): """Return the regularization loss from all Concrete Dropout layers.""" return (self.cd_enc1.regularization_loss + self.cd_enc2.regularization_loss + self.cd_enc3.regularization_loss + self.cd_enc4.regularization_loss + self.cd_enc5.regularization_loss + self.cd_dec1.regularization_loss + self.cd_dec2.regularization_loss + self.cd_dec3.regularization_loss + self.cd_dec4.regularization_loss) def get_dropout_rates(self): """Return the dropout rate for every layer.""" return { 'enc1': self.cd_enc1.get_dropout_rate(), 'enc2': self.cd_enc2.get_dropout_rate(), 'enc3': self.cd_enc3.get_dropout_rate(), 'enc4': self.cd_enc4.get_dropout_rate(), 'enc5': self.cd_enc5.get_dropout_rate(), 'dec1': self.cd_dec1.get_dropout_rate(), 'dec2': self.cd_dec2.get_dropout_rate(), 'dec3': self.cd_dec3.get_dropout_rate(), 'dec4': self.cd_dec4.get_dropout_rate() } if __name__ == "__main__": device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = UNet(3, bilinear=False).to(device) print(summary(model, (3, 432, 352))) # Example input. x = torch.randn(1, 3, 800, 800).to(device) output = model(x) print("Output mean M:", output['M_mean'].shape) print("Output variance M:", output['M_var'].shape) print("Output mean D:", output['D_mean'].shape) print("Output variance D:", output['D_var'].shape) # Print the dropout rate for every layer. dropout_rates = model.get_dropout_rates() for layer, rate in dropout_rates.items(): print(f"{layer} dropout rate: {rate:.4f}")