Update configuration_trillion.py

configuration_trillion.py

@@ -174,6 +174,7 @@ class TrillionConfig(PretrainedConfig):
         rope_scaling=None,
         global_attention_freq=4,
         attn_temperature_tuning=True,
+        sliding_window=4096,
         floor_scale=1.0,
         attn_scale=1.0,
         attention_bias=False,
@@ -183,6 +184,7 @@ class TrillionConfig(PretrainedConfig):
         **kwargs,
     ):
         self.attn_temperature_tuning = attn_temperature_tuning
+        self.sliding_window = sliding_window
         self.attn_scale = attn_scale
         self.floor_scale = floor_scale
         self.global_attention_freq = global_attention_freq
@@ -215,6 +217,10 @@ class TrillionConfig(PretrainedConfig):
             self.rope_scaling["rope_type"] = self.rope_scaling["type"]
         rope_config_validation(self)
 
+        self.layer_types = [
+            "sliding_attention" if bool((i + 1) % self.global_attention_freq) else "full_attention" for i in range(self.num_hidden_layers)
+        ]
+
         super().__init__(
             pad_token_id=pad_token_id,
             bos_token_id=bos_token_id,