# coding=utf-8 ############################ ''' The implement codes of FNO were taken and modified from: https://github.com/neuraloperator/physics_informed ''' import numpy as np import torch import torch.nn as nn from onescience.utils.pdenneval.pino_utils import add_padding, remove_padding, add_padding2, remove_padding2, add_padding3, remove_padding3, _get_act @torch.jit.script def compl_mul1d(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: # (batch, in_channel, x ), (in_channel, out_channel, x) -> (batch, out_channel, x) res = torch.einsum("bix,iox->box", a, b) return res @torch.jit.script def compl_mul2d(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: # (batch, in_channel, x,y,t ), (in_channel, out_channel, x,y,t) -> (batch, out_channel, x,y,t) res = torch.einsum("bixy,ioxy->boxy", a, b) return res @torch.jit.script def compl_mul3d(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: res = torch.einsum("bixyz,ioxyz->boxyz", a, b) return res ################################################################ # 1d fourier layer ################################################################ class SpectralConv1d(nn.Module): def __init__(self, in_channels, out_channels, modes1): super(SpectralConv1d, self).__init__() """ 1D Fourier layer. It does FFT, linear transform, and Inverse FFT. """ self.in_channels = in_channels self.out_channels = out_channels # Number of Fourier modes to multiply, at most floor(N/2) + 1 self.modes1 = modes1 self.scale = (1 / (in_channels*out_channels)) self.weights1 = nn.Parameter( self.scale * torch.rand(in_channels, out_channels, self.modes1, dtype=torch.cfloat)) def forward(self, x): batchsize = x.shape[0] # Compute Fourier coeffcients up to factor of e^(- something constant) x_ft = torch.fft.rfftn(x, dim=[2]) # Multiply relevant Fourier modes out_ft = torch.zeros(batchsize, self.in_channels, x.size(-1)//2 + 1, device=x.device, dtype=torch.cfloat) out_ft[:, :, :self.modes1] = compl_mul1d(x_ft[:, :, :self.modes1], self.weights1) # Return to physical space x = torch.fft.irfftn(out_ft, s=[x.size(-1)], dim=[2]) return x ################################################################ # 2d fourier layer ################################################################ class SpectralConv2d(nn.Module): def __init__(self, in_channels, out_channels, modes1, modes2): super(SpectralConv2d, self).__init__() self.in_channels = in_channels self.out_channels = out_channels # Number of Fourier modes to multiply, at most floor(N/2) + 1 self.modes1 = modes1 self.modes2 = modes2 self.scale = (1 / (in_channels * out_channels)) self.weights1 = nn.Parameter( self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, dtype=torch.cfloat)) self.weights2 = nn.Parameter( self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, dtype=torch.cfloat)) def forward(self, x): batchsize = x.shape[0] size1 = x.shape[-2] size2 = x.shape[-1] # Compute Fourier coeffcients up to factor of e^(- something constant) x_ft = torch.fft.rfftn(x, dim=[2, 3]) # Multiply relevant Fourier modes out_ft = torch.zeros(batchsize, self.out_channels, x.size(-2), x.size(-1) // 2 + 1, device=x.device, dtype=torch.cfloat) out_ft[:, :, :self.modes1, :self.modes2] = \ compl_mul2d(x_ft[:, :, :self.modes1, :self.modes2], self.weights1) out_ft[:, :, -self.modes1:, :self.modes2] = \ compl_mul2d(x_ft[:, :, -self.modes1:, :self.modes2], self.weights2) # Return to physical space x = torch.fft.irfftn(out_ft, s=(x.size(-2), x.size(-1)), dim=[2, 3]) return x class SpectralConv3d(nn.Module): def __init__(self, in_channels, out_channels, modes1, modes2, modes3): super(SpectralConv3d, self).__init__() self.in_channels = in_channels self.out_channels = out_channels self.modes1 = modes1 #Number of Fourier modes to multiply, at most floor(N/2) + 1 self.modes2 = modes2 self.modes3 = modes3 self.scale = (1 / (in_channels * out_channels)) self.weights1 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, dtype=torch.cfloat)) self.weights2 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, dtype=torch.cfloat)) self.weights3 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, dtype=torch.cfloat)) self.weights4 = nn.Parameter(self.scale * torch.rand(in_channels, out_channels, self.modes1, self.modes2, self.modes3, dtype=torch.cfloat)) def forward(self, x): batchsize = x.shape[0] # Compute Fourier coeffcients up to factor of e^(- something constant) x_ft = torch.fft.rfftn(x, dim=[2,3,4]) z_dim = min(x_ft.shape[4], self.modes3) # Multiply relevant Fourier modes out_ft = torch.zeros(batchsize, self.out_channels, x_ft.shape[2], x_ft.shape[3], self.modes3, device=x.device, dtype=torch.cfloat) # if x_ft.shape[4] > self.modes3, truncate; if x_ft.shape[4] < self.modes3, add zero padding coeff = torch.zeros(batchsize, self.in_channels, self.modes1, self.modes2, self.modes3, device=x.device, dtype=torch.cfloat) coeff[..., :z_dim] = x_ft[:, :, :self.modes1, :self.modes2, :z_dim] out_ft[:, :, :self.modes1, :self.modes2, :] = compl_mul3d(coeff, self.weights1) coeff = torch.zeros(batchsize, self.in_channels, self.modes1, self.modes2, self.modes3, device=x.device, dtype=torch.cfloat) coeff[..., :z_dim] = x_ft[:, :, -self.modes1:, :self.modes2, :z_dim] out_ft[:, :, -self.modes1:, :self.modes2, :] = compl_mul3d(coeff, self.weights2) coeff = torch.zeros(batchsize, self.in_channels, self.modes1, self.modes2, self.modes3, device=x.device, dtype=torch.cfloat) coeff[..., :z_dim] = x_ft[:, :, :self.modes1, -self.modes2:, :z_dim] out_ft[:, :, :self.modes1, -self.modes2:, :] = compl_mul3d(coeff, self.weights3) coeff = torch.zeros(batchsize, self.in_channels, self.modes1, self.modes2, self.modes3, device=x.device, dtype=torch.cfloat) coeff[..., :z_dim] = x_ft[:, :, -self.modes1:, -self.modes2:, :z_dim] out_ft[:, :, -self.modes1:, -self.modes2:, :] = compl_mul3d(coeff, self.weights4) #Return to physical space x = torch.fft.irfftn(out_ft, s=(x.size(2), x.size(3), x.size(4)), dim=[2,3,4]) return x class FourierBlock(nn.Module): def __init__(self, in_channels, out_channels, modes1, modes2, modes3, act='tanh'): super(FourierBlock, self).__init__() self.in_channel = in_channels self.out_channel = out_channels self.speconv = SpectralConv3d(in_channels, out_channels, modes1, modes2, modes3) self.linear = nn.Conv1d(in_channels, out_channels, 1) if act in ['tanh','gelu','none']: self.act=_get_act(act) else: raise ValueError(f'{act} is not supported') def forward(self, x): ''' input x: (batchsize, channel width, x_grid, y_grid, t_grid) ''' x1 = self.speconv(x) x2 = self.linear(x.view(x.shape[0], self.in_channel, -1)) out = x1 + x2.view(x.shape[0], self.out_channel, x.shape[2], x.shape[3], x.shape[4]) if self.act is not None: out = self.act(out) return out class FNO1d(nn.Module): def __init__(self, modes, width=32, layers=None, fc_dim=128, in_dim=2, out_dim=1, act='relu', pad_ratio=[0.,0.1]): super(FNO1d, self).__init__() """ The overall network. It contains several layers of the Fourier layer. 1. Lift the input to the desire channel dimension by self.fc0 . 2. 4 layers of the integral operators u' = (W + K)(u). W defined by self.w; K defined by self.conv . 3. Project from the channel space to the output space by self.fc1 and self.fc2 . input: the solution of the initial condition and location (a(x), x) input shape: (batchsize, x=s, c=2) output: the solution of a later timestep output shape: (batchsize, x=s, c=1) """ self.modes1 = modes self.width = width self.pad_ratio = pad_ratio if layers is None: layers = [width] * 4 self.fc0 = nn.Linear(in_dim, layers[0]) # input channel is 2: (a(x), x) self.sp_convs = nn.ModuleList([SpectralConv1d( in_size, out_size, num_modes) for in_size, out_size, num_modes in zip(layers, layers[1:], self.modes1)]) self.ws = nn.ModuleList([nn.Conv1d(in_size, out_size, 1) for in_size, out_size in zip(layers, layers[1:])]) self.fc1 = nn.Linear(layers[-1], fc_dim) self.fc2 = nn.Linear(fc_dim, out_dim) self.act = _get_act(act) def forward(self, x): length = len(self.ws) size_1= x.shape[1] if max(self.pad_ratio) > 0: num_pad = [round(size_1 * i) for i in self.pad_ratio] else: num_pad = [0., 0.] x = self.fc0(x) x = x.permute(0, 2, 1) x = add_padding(x,num_pad) for i, (speconv, w) in enumerate(zip(self.sp_convs, self.ws)): x1 = speconv(x) x2 = w(x) x = x1 + x2 if i != length - 1: x = self.act(x) x = remove_padding(x,num_pad) x = x.permute(0, 2, 1) x = self.fc1(x) x = self.act(x) x = self.fc2(x) return x class FNO2d(nn.Module): def __init__(self, modes1, modes2, width=64, fc_dim=128, layers=None, in_dim=3, out_dim=1, act='gelu', pad_ratio=[0., 0.1]): super(FNO2d, self).__init__() """ Args: - modes1: list of int, number of modes in first dimension in each layer - modes2: list of int, number of modes in second dimension in each layer - width: int, optional, if layers is None, it will be initialized as [width] * [len(modes1) + 1] - in_dim: number of input channels - out_dim: number of output channels - act: activation function, {tanh, gelu, relu, leaky_relu}, default: gelu - pad_ratio: list of float, or float; portion of domain to be extended. If float, paddings are added to the right. If list, paddings are added to both sides. pad_ratio[0] pads left, pad_ratio[1] pads right. """ if isinstance(pad_ratio, float): pad_ratio = [pad_ratio, pad_ratio] else: assert len(pad_ratio) == 2, 'Cannot add padding in more than 2 directions' self.modes1 = modes1 self.modes2 = modes2 self.pad_ratio = pad_ratio # input channel is 3: (a(x, y), x, y) if layers is None: self.layers = [width] * (len(modes1) + 1) else: self.layers = layers self.fc0 = nn.Linear(in_dim, self.layers[0]) self.sp_convs = nn.ModuleList([SpectralConv2d( in_size, out_size, mode1_num, mode2_num) for in_size, out_size, mode1_num, mode2_num in zip(self.layers, self.layers[1:], self.modes1, self.modes2)]) self.ws = nn.ModuleList([nn.Conv1d(in_size, out_size, 1) for in_size, out_size in zip(self.layers, self.layers[1:])]) self.fc1 = nn.Linear(self.layers[-1], fc_dim) self.fc2 = nn.Linear(fc_dim, self.layers[-1]) self.fc3 = nn.Linear(self.layers[-1], out_dim) self.act = _get_act(act) def forward(self, x): ''' Args: - x : (batch size, x_grid, y_grid, 2) Returns: - x: (batch size, x_grid, y_grid, 1) ''' size_1, size_2 = x.shape[1], x.shape[2] if max(self.pad_ratio) > 0: num_pad1 = [round(i * size_1) for i in self.pad_ratio] num_pad2 = [round(i * size_2) for i in self.pad_ratio] else: num_pad1 = num_pad2 = [0.,0.] length = len(self.ws) batchsize = x.shape[0] x = self.fc0(x) x = x.permute(0, 3, 1, 2) # B, C, X, Y x = add_padding2(x, num_pad1, num_pad2) size_x, size_y = x.shape[-2], x.shape[-1] for i, (speconv, w) in enumerate(zip(self.sp_convs, self.ws)): x1 = speconv(x) x2 = w(x.view(batchsize, self.layers[i], -1)).view(batchsize, self.layers[i+1], size_x, size_y) x = x1 + x2 if i != length - 1: x = self.act(x) x = remove_padding2(x, num_pad1, num_pad2) x = x.permute(0, 2, 3, 1) x = self.fc1(x) x = self.act(x) x = self.fc2(x) x = self.act(x) x = self.fc3(x) return x class FNO3d(nn.Module): def __init__(self, modes1, modes2, modes3, width=16, fc_dim=128, layers=None, in_dim=4, out_dim=1, act='gelu', pad_ratio=[0., 0.05]): ''' Args: modes1: list of int, first dimension maximal modes for each layer modes2: list of int, second dimension maximal modes for each layer modes3: list of int, third dimension maximal modes for each layer layers: list of int, channels for each layer fc_dim: dimension of fully connected layers in_dim: int, input dimension out_dim: int, output dimension act: {tanh, gelu, relu, leaky_relu}, activation function pad_ratio: the ratio of the extended domain ''' super(FNO3d, self).__init__() if isinstance(pad_ratio, float): pad_ratio = [pad_ratio, pad_ratio] else: assert len(pad_ratio) == 2, 'Cannot add padding in more than 2 directions.' self.pad_ratio = pad_ratio self.modes1 = modes1 self.modes2 = modes2 self.modes3 = modes3 self.pad_ratio = pad_ratio if layers is None: self.layers = [width] * 4 else: self.layers = layers self.fc0 = nn.Linear(in_dim, self.layers[0]) self.sp_convs = nn.ModuleList([SpectralConv3d( in_size, out_size, mode1_num, mode2_num, mode3_num) for in_size, out_size, mode1_num, mode2_num, mode3_num in zip(self.layers, self.layers[1:], self.modes1, self.modes2, self.modes3)]) self.ws = nn.ModuleList([nn.Conv1d(in_size, out_size, 1) for in_size, out_size in zip(self.layers, self.layers[1:])]) self.fc1 = nn.Linear(self.layers[-1], fc_dim) self.fc2 = nn.Linear(fc_dim, out_dim) self.act = _get_act(act) def forward(self, x): ''' Args: x: (batchsize, x_grid, y_grid, t_grid, 3) Returns: u: (batchsize, x_grid, y_grid, t_grid, 1) ''' size_x,size_y,size_z = x.shape[1],x.shape[2],x.shape[3] if max(self.pad_ratio) > 0: num_pad1 = [round(size_x * i) for i in self.pad_ratio] num_pad2 = [round(size_y * i) for i in self.pad_ratio] num_pad3 = [round(size_z * i) for i in self.pad_ratio] else: num_pad1 = num_pad2 = num_pad3 = [0., 0.] length = len(self.ws) batchsize = x.shape[0] x = self.fc0(x) x = x.permute(0, 4, 1, 2, 3) x = add_padding3(x, num_pad1,num_pad2,num_pad3) size_x, size_y, size_z = x.shape[-3], x.shape[-2], x.shape[-1] for i, (speconv, w) in enumerate(zip(self.sp_convs, self.ws)): x1 = speconv(x) x2 = w(x.view(batchsize, self.layers[i], -1)).view(batchsize, self.layers[i+1], size_x, size_y, size_z) x = x1 + x2 if i != length - 1: x = self.act(x) x = remove_padding3(x, num_pad1,num_pad2,num_pad3) x = x.permute(0, 2, 3, 4, 1) x = self.fc1(x) x = self.act(x) x = self.fc2(x) return x