lamba_cpu: fix GatedMLP gate order (silu(a)*b -> y*silu(gate)) — CPU port now matches fork to 0.06-0.09 logit
Browse files- code/kod/lamba_cpu.py +3 -2
code/kod/lamba_cpu.py
CHANGED
|
@@ -174,8 +174,9 @@ class GatedMLP(nn.Module):
|
|
| 174 |
self.fc2 = nn.Linear(hidden, d, bias=False)
|
| 175 |
|
| 176 |
def forward(self, x):
|
| 177 |
-
|
| 178 |
-
|
|
|
|
| 179 |
|
| 180 |
|
| 181 |
class Block(nn.Module):
|
|
|
|
| 174 |
self.fc2 = nn.Linear(hidden, d, bias=False)
|
| 175 |
|
| 176 |
def forward(self, x):
|
| 177 |
+
# mamba_ssm GatedMLP: 1. yarı = değer, 2. yarı = gate → y * silu(gate)
|
| 178 |
+
y, gate = self.fc1(x).chunk(2, -1)
|
| 179 |
+
return self.fc2(y * F.silu(gate))
|
| 180 |
|
| 181 |
|
| 182 |
class Block(nn.Module):
|