kdirgul commited on
Commit
a36fa9e
·
verified ·
1 Parent(s): 0b3a752

lamba_cpu: fix GatedMLP gate order (silu(a)*b -> y*silu(gate)) — CPU port now matches fork to 0.06-0.09 logit

Browse files
Files changed (1) hide show
  1. code/kod/lamba_cpu.py +3 -2
code/kod/lamba_cpu.py CHANGED
@@ -174,8 +174,9 @@ class GatedMLP(nn.Module):
174
  self.fc2 = nn.Linear(hidden, d, bias=False)
175
 
176
  def forward(self, x):
177
- a, b = self.fc1(x).chunk(2, -1)
178
- return self.fc2(F.silu(a) * b)
 
179
 
180
 
181
  class Block(nn.Module):
 
174
  self.fc2 = nn.Linear(hidden, d, bias=False)
175
 
176
  def forward(self, x):
177
+ # mamba_ssm GatedMLP: 1. yarı = değer, 2. yarı = gate → y * silu(gate)
178
+ y, gate = self.fc1(x).chunk(2, -1)
179
+ return self.fc2(y * F.silu(gate))
180
 
181
 
182
  class Block(nn.Module):