Sync configuration_auristream.py from TuKoResearch/AuriStream100M_40Pred_BigAudioDataset_500k

configuration_auristream.py

@@ -5,14 +5,15 @@ class AuriStreamConfig(PretrainedConfig):
 
     def __init__(
         self,
-        n_layer=12,
-        n_head=12,
-        n_embd=768,
+        n_layer=48,
+        n_head=16,
+        n_embd=1280,
         vocab_size=8192,
         dropout=0.0,
         bias=False,
-        use_rope=True,
+        use_rope=False,
         n_pred_steps=20,
+        seq_len=4096,
         **kwargs
     ):
         self.n_layer = n_layer
@@ -23,4 +24,5 @@ class AuriStreamConfig(PretrainedConfig):
         self.bias = bias
         self.use_rope = use_rope
         self.n_pred_steps = n_pred_steps
+        self.seq_len = seq_len
         super().__init__(**kwargs)