OpenLab-NLP
/

model-prototype

Model card Files Files and versions

Yuchan commited on Nov 23

Commit

ebb1511

·

verified ·

1 Parent(s): 55eb46d

Update Mo.py

Files changed (1) hide show

Mo.py +1 -1

Mo.py CHANGED Viewed

@@ -137,7 +137,7 @@ class LoUScan(layers.Layer):
         self.norm = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
         self.norm1 = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
-        self.glu = SwiGLU(d_model, 3500)
     def call(self, x):
         x_f32 = tf.cast(x, tf.float32)

         self.norm = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
         self.norm1 = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
+        self.glu = SwiGLU(d_model, 320)
     def call(self, x):
         x_f32 = tf.cast(x, tf.float32)