Update 연구중.py

연구중.py

@@ -135,18 +135,14 @@ class MixerBlock(layers.Layer):
         self.dim = dim
 
         self.ln_token = layers.LayerNormalization(epsilon=1e-6)
-        self.ln_gate  = layers.LayerNormalization(epsilon=1e-6) # 이름 변경
         self.ln_channel = layers.LayerNormalization(epsilon=1e-6)
 
         # Token Mixer
-        self.token_fc1 = layers.Dense(seq_len * 2)
+        self.token_fc1 = layers.Dense(seq_len * 4)
         self.token_fc2 = layers.Dense(seq_len)
 
-        # Gating (Sigmoid) - Temperature 불필요
-        self.gate_dense = layers.Dense(1)
-
         # Channel Mixer
-        self.ch_fc1 = layers.Dense(self.dim * 4, activation='gelu')
+        self.ch_fc1 = layers.Dense(self.dim * 4)
         self.ch_fc2 = layers.Dense(self.dim)
 
     def call(self, x, training=None):
@@ -159,19 +155,11 @@ class MixerBlock(layers.Layer):
         y = tf.transpose(y_t, perm=[0, 2, 1])
         x = x + y
 
-        # 2. Scalar Gating (수정됨)
-        # Softmax의 1/N 희석 문제를 해결하기 위해 Sigmoid 사용
-        y = self.ln_gate(x)
-        gate = tf.nn.sigmoid(self.gate_dense(y)) # (B, L, 1) Range: 0~1
-        y = y * gate
-        x = x + y 
-
-        # 3. Channel Mixer
         y = self.ln_channel(x)
-        y = self.ch_fc1(y)
-        y = self.ch_fc2(y)
+        a, b = tf.split(self.ch_fc1(y), 2, axis=-1)
+        y = self.ch_fc2(a * tf.nn.gelu(b))
         x = x + y
-        
+
         return x