Update Mo.py

Mo.py

@@ -226,17 +226,6 @@ def smoothed_loss_keras(y_true, y_pred, eps=0.1):
     per_tok = per_tok * mask
     return tf.reduce_sum(per_tok) / (tf.reduce_sum(mask) + 1e-8)
 
-def masked_perplexity(y_true, y_pred, eps=0.1):
-    y_true = tf.cast(y_true, tf.int32)
-    mask = tf.cast(tf.not_equal(y_true, pad_id), tf.float32)
-    vocab = tf.shape(y_pred)[-1]
-    y_true_oh = tf.one_hot(y_true, depth=vocab, dtype=tf.float32)
-    y_true_ls = (1.0 - eps) * y_true_oh + eps / tf.cast(vocab, tf.float32)
-    log_probs = tf.nn.log_softmax(y_pred, axis=-1)
-    per_tok = -tf.reduce_sum(y_true_ls * log_probs, axis=-1)
-    per_tok = per_tok * mask
-    mean_loss = tf.reduce_sum(per_tok) / (tf.reduce_sum(mask) + 1e-8)
-    return tf.exp(mean_loss)
 
 with strategy.scope():
     model = LaSLM(vocab_size=vocab_size, max_seq_len=max_len, d_model=384, n_layers=3)
@@ -245,7 +234,7 @@ with strategy.scope():
     model.summary()
 
     optimizer = tf.keras.optimizers.Adam(1e-4, beta_1=0.9, beta_2=0.95, epsilon=1e-8, clipnorm=1.0)
-    model.compile(optimizer=optimizer, loss=smoothed_loss_keras, metrics=[masked_perplexity])
+    model.compile(optimizer=optimizer, loss=smoothed_loss_keras)
 
     # 학습
     history = model.fit(dist_dataset, epochs=1, steps_per_epoch=steps_per_epoch, verbose=1)