Update Mo.py

Mo.py

@@ -123,6 +123,7 @@ class SwiGLU(layers.Layer):
         x_proj = self.proj(x)
         x_val, x_gate = tf.split(x_proj, 2, axis=-1)
         return self.out(x_val * tf.nn.silu(x_gate))
+        
 
 class LoU(layers.Layer):
     def __init__(self, d_model, clip_value=5.0, eps=1e-6):
@@ -196,7 +197,7 @@ class Block(layers.Layer):
         x = self.lo(x)
         return x
 
-class CumaLM(tf.keras.Model):
+class ReLM(tf.keras.Model):
     def __init__(self, vocab_size, max_seq_len, d_model, n_layers, dropout_rate=0.1):
         super().__init__()
         self.token_embedding = layers.Embedding(vocab_size, d_model)
@@ -215,6 +216,8 @@ class CumaLM(tf.keras.Model):
         logits = tf.matmul(x, embedding_matrix, transpose_b=True)
         return tf.cast(logits, tf.float32)
 
+
+
 def smoothed_loss_keras(y_true, y_pred, eps=0.1):
     y_true = tf.cast(y_true, tf.int32)
     mask = tf.cast(tf.not_equal(y_true, pad_id), tf.float32)
@@ -242,7 +245,7 @@ def masked_perplexity(y_true, y_pred, eps=0.1):
 # 모델 생성 & 컴파일
 # =======================
 with strategy.scope():
-    model = CumaLM(vocab_size=vocab_size, max_seq_len=max_len, d_model=256, n_layers=1)
+    model = ReLM(vocab_size=vocab_size, max_seq_len=max_len, d_model=256, n_layers=1)
     dummy_input = tf.zeros((batch_size, max_len), dtype=tf.int32)
     _ = model(dummy_input, training=False)
     model.summary()