Yuchan
commited on
Update Mo.py
Browse files
Mo.py
CHANGED
|
@@ -196,7 +196,7 @@ class Block(layers.Layer):
|
|
| 196 |
x = self.lo(x)
|
| 197 |
return x
|
| 198 |
|
| 199 |
-
class
|
| 200 |
def __init__(self, vocab_size, max_seq_len, d_model, n_layers, dropout_rate=0.1):
|
| 201 |
super().__init__()
|
| 202 |
self.token_embedding = layers.Embedding(vocab_size, d_model)
|
|
@@ -242,7 +242,7 @@ def masked_perplexity(y_true, y_pred, eps=0.1):
|
|
| 242 |
# 모델 생성 & 컴파일
|
| 243 |
# =======================
|
| 244 |
with strategy.scope():
|
| 245 |
-
model =
|
| 246 |
dummy_input = tf.zeros((batch_size, max_len), dtype=tf.int32)
|
| 247 |
_ = model(dummy_input, training=False)
|
| 248 |
model.summary()
|
|
|
|
| 196 |
x = self.lo(x)
|
| 197 |
return x
|
| 198 |
|
| 199 |
+
class CumaLM(tf.keras.Model):
|
| 200 |
def __init__(self, vocab_size, max_seq_len, d_model, n_layers, dropout_rate=0.1):
|
| 201 |
super().__init__()
|
| 202 |
self.token_embedding = layers.Embedding(vocab_size, d_model)
|
|
|
|
| 242 |
# 모델 생성 & 컴파일
|
| 243 |
# =======================
|
| 244 |
with strategy.scope():
|
| 245 |
+
model = CumaLM(vocab_size=vocab_size, max_seq_len=max_len, d_ff=768, n_layers=12)
|
| 246 |
dummy_input = tf.zeros((batch_size, max_len), dtype=tf.int32)
|
| 247 |
_ = model(dummy_input, training=False)
|
| 248 |
model.summary()
|