import torch import torch.nn as nn from torchvision import models import torch.nn.functional as F class sobel_net(nn.Module): def __init__(self): super().__init__() self.conv_opx = nn.Conv2d(1, 1, 3, bias=False) self.conv_opy = nn.Conv2d(1, 1, 3, bias=False) sobel_kernelx = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], dtype='float32').reshape((1, 1, 3, 3)) sobel_kernely = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]], dtype='float32').reshape((1, 1, 3, 3)) self.conv_opx.weight.data = torch.from_numpy(sobel_kernelx) self.conv_opy.weight.data = torch.from_numpy(sobel_kernely) for p in self.parameters(): p.requires_grad = False def forward(self, im): # input rgb x = (0.299 * im[:, 0, :, :] + 0.587 * im[:, 1, :, :] + 0.114 * im[:, 2, :, :]).unsqueeze(1) # rgb2gray gradx = self.conv_opx(x) grady = self.conv_opy(x) x = (gradx ** 2 + grady ** 2) ** 0.5 x = (x - x.min()) / (x.max() - x.min()) x = F.pad(x, (1, 1, 1, 1)) x = torch.cat([im, x], dim=1) return x class conv_block(nn.Module): def __init__(self, ch_in, ch_out): super(conv_block, self).__init__() self.conv = nn.Sequential( nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(ch_out), nn.ReLU(inplace=True), nn.Conv2d(ch_out, ch_out, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(ch_out), nn.ReLU(inplace=True) ) def forward(self, x): x = self.conv(x) return x class up_conv(nn.Module): def __init__(self, ch_in, ch_out): super(up_conv, self).__init__() self.up = nn.Sequential( nn.Upsample(scale_factor=2), nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(ch_out), nn.ReLU(inplace=True) ) def forward(self, x): x = self.up(x) return x class Recurrent_block(nn.Module): def __init__(self, ch_out, t=2): super(Recurrent_block, self).__init__() self.t = t self.ch_out = ch_out self.conv = nn.Sequential( nn.Conv2d(ch_out, ch_out, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(ch_out), nn.ReLU(inplace=True) ) def forward(self, x): for i in range(self.t): if i == 0: x1 = self.conv(x) x1 = self.conv(x + x1) return x1 class RRCNN_block(nn.Module): def __init__(self, ch_in, ch_out, t=2): super(RRCNN_block, self).__init__() self.RCNN = nn.Sequential( Recurrent_block(ch_out, t=t), Recurrent_block(ch_out, t=t) ) self.Conv_1x1 = nn.Conv2d(ch_in, ch_out, kernel_size=1, stride=1, padding=0) def forward(self, x): x = self.Conv_1x1(x) x1 = self.RCNN(x) return x + x1 class single_conv(nn.Module): def __init__(self, ch_in, ch_out): super(single_conv, self).__init__() self.conv = nn.Sequential( nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=1, padding=1, bias=True), nn.BatchNorm2d(ch_out), nn.ReLU(inplace=True) ) def forward(self, x): x = self.conv(x) return x class Attention_block(nn.Module): def __init__(self, F_g, F_l, F_int): super(Attention_block, self).__init__() self.W_g = nn.Sequential( nn.Conv2d(F_g, F_int, kernel_size=1, stride=1, padding=0, bias=True), nn.BatchNorm2d(F_int) ) self.W_x = nn.Sequential( nn.Conv2d(F_l, F_int, kernel_size=1, stride=1, padding=0, bias=True), nn.BatchNorm2d(F_int) ) self.psi = nn.Sequential( nn.Conv2d(F_int, 1, kernel_size=1, stride=1, padding=0, bias=True), nn.BatchNorm2d(1), nn.Sigmoid() ) self.relu = nn.ReLU(inplace=True) def forward(self, g, x): g1 = self.W_g(g) x1 = self.W_x(x) psi = self.relu(g1 + x1) psi = self.psi(psi) return x * psi class U_Net(nn.Module): def __init__(self, img_ch=3, output_ch=1): super(U_Net, self).__init__() self.Maxpool = nn.MaxPool2d(kernel_size=2, stride=2) self.Conv1 = conv_block(ch_in=img_ch, ch_out=64) self.Conv2 = conv_block(ch_in=64, ch_out=128) self.Conv3 = conv_block(ch_in=128, ch_out=256) self.Conv4 = conv_block(ch_in=256, ch_out=512) self.Conv5 = conv_block(ch_in=512, ch_out=1024) self.Up5 = up_conv(ch_in=1024, ch_out=512) self.Up_conv5 = conv_block(ch_in=1024, ch_out=512) self.Up4 = up_conv(ch_in=512, ch_out=256) self.Up_conv4 = conv_block(ch_in=512, ch_out=256) self.Up3 = up_conv(ch_in=256, ch_out=128) self.Up_conv3 = conv_block(ch_in=256, ch_out=128) self.Up2 = up_conv(ch_in=128, ch_out=64) self.Up_conv2 = conv_block(ch_in=128, ch_out=64) self.Conv_1x1 = nn.Conv2d(64, output_ch, kernel_size=1, stride=1, padding=0, bias=False) def forward(self, x): # encoding path x1 = self.Conv1(x) x2 = self.Maxpool(x1) x2 = self.Conv2(x2) x3 = self.Maxpool(x2) x3 = self.Conv3(x3) x4 = self.Maxpool(x3) x4 = self.Conv4(x4) x5 = self.Maxpool(x4) x5 = self.Conv5(x5) # decoding + concat path d5 = self.Up5(x5) d5 = torch.cat((x4, d5), dim=1) d5 = self.Up_conv5(d5) d4 = self.Up4(d5) d4 = torch.cat((x3, d4), dim=1) d4 = self.Up_conv4(d4) d3 = self.Up3(d4) d3 = torch.cat((x2, d3), dim=1) d3 = self.Up_conv3(d3) d2 = self.Up2(d3) d2 = torch.cat((x1, d2), dim=1) d2 = self.Up_conv2(d2) out = self.Conv_1x1(d2) out = torch.sigmoid(out) return out class U_Net_mini(nn.Module): def __init__(self, img_ch=3, output_ch=1): super(U_Net_mini, self).__init__() self.Maxpool = nn.MaxPool2d(kernel_size=2, stride=2) self.Conv1 = conv_block(ch_in=img_ch, ch_out=32) self.Conv2 = conv_block(ch_in=32, ch_out=64) self.Conv3 = conv_block(ch_in=64, ch_out=128) self.Conv4 = conv_block(ch_in=128, ch_out=256) self.Conv5 = conv_block(ch_in=256, ch_out=512) self.Up5 = up_conv(ch_in=512, ch_out=256) self.Up_conv5 = conv_block(ch_in=512, ch_out=256) self.Up4 = up_conv(ch_in=256, ch_out=128) self.Up_conv4 = conv_block(ch_in=256, ch_out=128) self.Up3 = up_conv(ch_in=128, ch_out=64) self.Up_conv3 = conv_block(ch_in=128, ch_out=64) self.Up2 = up_conv(ch_in=64, ch_out=32) self.Up_conv2 = conv_block(ch_in=64, ch_out=32) self.Conv_1x1 = nn.Conv2d(32, output_ch, kernel_size=1, stride=1, padding=0, bias=False) def forward(self, x): # encoding path x1 = self.Conv1(x) x2 = self.Maxpool(x1) x2 = self.Conv2(x2) x3 = self.Maxpool(x2) x3 = self.Conv3(x3) x4 = self.Maxpool(x3) x4 = self.Conv4(x4) x5 = self.Maxpool(x4) x5 = self.Conv5(x5) # decoding + concat path d5 = self.Up5(x5) d5 = torch.cat((x4, d5), dim=1) d5 = self.Up_conv5(d5) d4 = self.Up4(d5) d4 = torch.cat((x3, d4), dim=1) d4 = self.Up_conv4(d4) d3 = self.Up3(d4) d3 = torch.cat((x2, d3), dim=1) d3 = self.Up_conv3(d3) d2 = self.Up2(d3) d2 = torch.cat((x1, d2), dim=1) d2 = self.Up_conv2(d2) out = self.Conv_1x1(d2) out = torch.sigmoid(out) return d4, out class AttU_Net(nn.Module): def __init__(self, img_ch=3, output_ch=1, need_feature_maps=False): super(AttU_Net, self).__init__() self.conv1_ = nn.Sequential(nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3), nn.BatchNorm2d(64), nn.ReLU(inplace=True)) self.Maxpool = nn.MaxPool2d(kernel_size=2, stride=2) self.Conv1 = conv_block(ch_in=64, ch_out=64) self.Conv2 = conv_block(ch_in=64, ch_out=128) self.Conv3 = conv_block(ch_in=128, ch_out=256) self.Conv4 = conv_block(ch_in=256, ch_out=512) self.Conv5 = conv_block(ch_in=512, ch_out=1024) self.Up5 = up_conv(ch_in=1024, ch_out=512) self.Att5 = Attention_block(F_g=512, F_l=512, F_int=256) self.Up_conv5 = conv_block(ch_in=1024, ch_out=512) self.Up4 = up_conv(ch_in=512, ch_out=256) self.Att4 = Attention_block(F_g=256, F_l=256, F_int=128) self.Up_conv4 = conv_block(ch_in=512, ch_out=256) self.Up3 = up_conv(ch_in=256, ch_out=128) self.Att3 = Attention_block(F_g=128, F_l=128, F_int=64) self.Up_conv3 = conv_block(ch_in=256, ch_out=128) self.Up2 = up_conv(ch_in=128, ch_out=64) self.Att2 = Attention_block(F_g=64, F_l=64, F_int=32) self.Up_conv2 = conv_block(ch_in=128, ch_out=64) self.Conv_1x1 = nn.Conv2d(64, output_ch, kernel_size=1, stride=1, padding=0) self.need_feature_maps = need_feature_maps # self.loc_xy = (torch.stack(torch.meshgrid([torch.arange(0, 256), torch.arange(0, 256)])).permute(0, 2, 1).unsqueeze(0).float() - 127.5) / 127.5 # 1*2*256*256 def forward(self, x): # encoding path # batch = x.size(0) # if self.need_feature_maps: # x = torch.cat((x, self.loc_xy.repeat(batch, 1, 1, 1).cuda()), dim=1) x1 = self.conv1_(x) x1 = self.Conv1(x1) x2 = self.Maxpool(x1) x2 = self.Conv2(x2) x3 = self.Maxpool(x2) x3 = self.Conv3(x3) x4 = self.Maxpool(x3) x4 = self.Conv4(x4) x5 = self.Maxpool(x4) x5 = self.Conv5(x5) # decoding + concat path d5 = self.Up5(x5) x4 = self.Att5(g=d5, x=x4) d5 = torch.cat((x4, d5), dim=1) d5 = self.Up_conv5(d5) d4 = self.Up4(d5) x3 = self.Att4(g=d4, x=x3) d4 = torch.cat((x3, d4), dim=1) d4 = self.Up_conv4(d4) d3 = self.Up3(d4) x2 = self.Att3(g=d3, x=x2) d3 = torch.cat((x2, d3), dim=1) d3 = self.Up_conv3(d3) d2 = self.Up2(d3) x1 = self.Att2(g=d2, x=x1) d2 = torch.cat((x1, d2), dim=1) d2 = self.Up_conv2(d2) wc = self.Conv_1x1(d2) if self.need_feature_maps: return d2, wc else: return bm class Doc_UNet(nn.Module): def __init__(self): super(Doc_UNet, self).__init__() self.U_net1 = AttU_Net(3, 3, need_feature_maps=True) self.U_net2 = U_Net(64 + 3 + 2, 2, need_feature_maps=False) self.htan = nn.Hardtanh(0, 1.0) self.f_activation = nn.Hardtanh() self.loc_xy = (torch.stack(torch.meshgrid([torch.arange(0, 128), torch.arange(0, 128)])).permute(0, 2, 1).unsqueeze( 0).float() - 63.5) / 63.5 # 1*2*256*256 def forward(self, x): batch = x.size(0) feature_maps, wc = self.U_net1(x) wc = self.htan(wc) x = torch.cat((self.loc_xy.repeat(batch, 1, 1, 1).cuda(), wc, feature_maps), dim=1) bm = self.U_net2(x) bm = self.f_activation(bm) return wc, bm def get_parameter_number(net): total_num = sum(p.numel() for p in net.parameters()) trainable_num = sum(p.numel() for p in net.parameters() if p.requires_grad) return {'Total': total_num, 'Trainable': trainable_num} if __name__ == '__main__': net = U2NET(3, 1).cuda() print(get_parameter_number(net)) # 69090500 加attention后69442032 with torch.no_grad(): inputs = torch.zeros(1, 3, 256, 256).cuda() outs = net(inputs) print(outs[0].shape) # torch.Size([2, 3, 256, 256]) torch.Size([2, 2, 256, 256])