Update modeling_neollm.py

modeling_neollm.py

@@ -2353,7 +2353,7 @@ class NeoLLMMLP(nn.Module):
             config.intermediate_size, config.hidden_size,
             bias=False, use_row_multiplier=True, use_column_multiplier=True,
         )
-        self.act_fn  = PolyNorm(exclusive_init=0.05)
+        self.act_fn  = PolyNorm(exclusive_init=0.15)
         self.dropout = nn.Dropout(config.dropout_rate)
 
     def forward(