Update modeling_phi.py

modeling_phi.py

@@ -756,8 +756,13 @@ class ParallelBlock(nn.Module):
         self.resid_dropout = nn.Dropout(config.resid_pdrop)
         self.block_idx = block_idx
 
+        if block_idx % 2 == 0:
+            n_inner = 4 * config.n_embd
+        else:
+            n_inner = 1024
+
         self.mixer = MHA(config, layer_idx=block_idx)
-        self.mlp = MLP(config)
+        self.mlp = MLP(config, n_inner)
 
     def forward(
         self,