Update modeling_switchgpt2.py

modeling_switchgpt2.py

@@ -171,22 +171,22 @@ class Attention(nn.Module):
         outputs = [a, present] + attn_outputs[1:]
         return outputs  # a, present, (attentions)
 
-
-class MLP(nn.Module):
-    def __init__(self, n_state, config):  # in MLP: n_state=3072 (4 * n_embd)
-        super().__init__()
-        nx = config.n_embd
-        # self.c_fc = Conv1D(n_state, nx)
-        # self.c_proj = Conv1D(nx, n_state)
-        self.c_fc = nn.Linear(nx, n_state)
-        self.c_proj = nn.Linear(n_state, nx)
-        self.act = ACT2FN[config.activation_function]
-        self.dropout = nn.Dropout(config.resid_pdrop)
-
-    def forward(self, x):
-        h = self.act(self.c_fc(x))
-        h2 = self.c_proj(h)
-        return self.dropout(h2)
+# the old MLP class
+# class MLP(nn.Module):
+#     def __init__(self, n_state, config):  # in MLP: n_state=3072 (4 * n_embd)
+#         super().__init__()
+#         nx = config.n_embd
+#         # self.c_fc = Conv1D(n_state, nx)
+#         # self.c_proj = Conv1D(nx, n_state)
+#         self.c_fc = nn.Linear(nx, n_state)
+#         self.c_proj = nn.Linear(n_state, nx)
+#         self.act = ACT2FN[config.activation_function]
+#         self.dropout = nn.Dropout(config.resid_pdrop)
+
+#     def forward(self, x):
+#         h = self.act(self.c_fc(x))
+#         h2 = self.c_proj(h)
+#         return self.dropout(h2)
 
 
 class Block(nn.Module):