File size: 469 Bytes
ad63418 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | import torch
import torch.nn as nn
import torch.nn.functional as F
class PositionWiseFFN(nn.Module):
def __init__(self, d_model, d_ff, dropout=0.1):
super().__init__()
self.linear1 = nn.Linear(d_model, d_ff)
self.linear2 = nn.Linear(d_ff, d_model)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
x = self.linear1(x)
x = F.relu(x)
x = self.dropout(x)
x = self.linear2(x)
return x |