Update Mo.py

Mo.py

@@ -69,7 +69,7 @@ vocab_size = sp.get_piece_size()
 print(f"✅ Vocabulary size: {vocab_size}")
 
 max_len = 512
-batch_size = 128
+batch_size = 256
 
 def text_to_ids(text):
     return sp.encode(text, out_type=int)
@@ -117,7 +117,7 @@ with strategy.scope():
 class SwiGLU(layers.Layer):
     def __init__(self, d_model, d_ff):
         super().__init__()
-        self.proj = layers.Dense(d_ff)
+        self.proj = layers.Dense(2048)
         self.out = layers.Dense(d_model)
     def call(self, x):
         x_proj = self.proj(x)
@@ -257,7 +257,7 @@ def masked_perplexity(y_true, y_pred, eps=0.1):
 # 모델 생성 & 컴파일
 # =======================
 with strategy.scope():
-    model = ReLM(vocab_size=vocab_size, max_seq_len=max_len, d_model=256, n_layers=1)
+    model = ReLM(vocab_size=vocab_size, max_seq_len=max_len, d_model=512, n_layers=9)
     dummy_input = tf.zeros((batch_size, max_len), dtype=tf.int32)
     _ = model(dummy_input, training=False)
     model.summary()