Update AlphaS2S.py

AlphaS2S.py

@@ -13,7 +13,7 @@ tf.get_logger().setLevel("ERROR")
 SEED = 42
 tf.random.set_seed(SEED)
 np.random.seed(SEED)
-max_len = 220 # 기존 코드에서 200으로 설정됨
+max_len = 128 # 기존 코드에서 200으로 설정됨
 batch_size = 48
 
 # TPU 초기화 (기존 코드와 동일)
@@ -180,7 +180,7 @@ class SwiGLU(layers.Layer):
 class EncoderBlock(layers.Layer):
     def __init__(self, d_model, num_heads, dff, dropout=0.1):
         super().__init__()
-        self.mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model)
+        self.mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model//num_heads)
         self.ffn = SwiGLU(d_model, dff)
         self.norm1 = layers.LayerNormalization(epsilon=1e-6)
         self.norm2 = layers.LayerNormalization(epsilon=1e-6)
@@ -195,8 +195,8 @@ class EncoderBlock(layers.Layer):
 class DecoderBlock(layers.Layer):
     def __init__(self, d_model, num_heads, dff, dropout=0.1):
         super().__init__()
-        self.self_mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model)
-        self.cross_mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model)
+        self.self_mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model//num_heads)
+        self.cross_mha = layers.MultiHeadAttention(num_heads=num_heads, key_dim=d_model//num_heads)
         self.ffn = SwiGLU(d_model, dff)
         self.norm1 = layers.LayerNormalization(epsilon=1e-6)
         self.norm2 = layers.LayerNormalization(epsilon=1e-6)
@@ -213,7 +213,7 @@ class DecoderBlock(layers.Layer):
         return self.norm3(out2 + ffn_out)
 
 class Transformer(tf.keras.Model):
-    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, max_len=256, dropout=0.1):
+    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, max_len=128, dropout=0.1):
         super().__init__()
         self.max_len = max_len
         self.d_model = d_model
@@ -271,7 +271,7 @@ def create_lr_schedule(initial_lr=5e-5, decay_steps=10000, decay_rate=0.9):
 
 with strategy.scope():
     # ⚠️ 수정: chat_vocab_size 대신 정의된 vocab_size 사용
-    chat_model = Transformer(num_layers=2, d_model=256, num_heads=4, dff=768, input_vocab_size=vocab_size, target_vocab_size=evocab_size, max_len=256, dropout=0.1)
+    chat_model = Transformer(num_layers=2, d_model=160, num_heads=4, dff=640, input_vocab_size=vocab_size, target_vocab_size=evocab_size, max_len=128, dropout=0.1)
     
     dummy_input = {
         "enc_inputs": tf.zeros((1, max_len), dtype=tf.int32),