Update Mo.py

Mo.py

@@ -137,7 +137,7 @@ class LoU(layers.Layer):
         self.norm = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
         self.norm1 = layers.LayerNormalization(epsilon=1e-5, dtype='float32')
         
-        self.glu = SwiGLU(d_model, 320)
+        self.glu = SwiGLU(d_model, 3500)
     def call(self, x):
         x_f32 = tf.cast(x, tf.float32)
         residual = x_f32
@@ -242,8 +242,8 @@ def create_lr_schedule(initial_lr=5e-5, decay_steps=10000, decay_rate=0.9):
 model = ReLM(
     vocab_size=vocab_size,
     max_seq_len=max_len,
-    d_model=256,
-    n_layers=1
+    d_model=700,
+    n_layers=16
 )
 
 # 옵티마이저 설정