adjust tokenizer config

tokenization_aria.py

@@ -32,16 +32,14 @@ class AriaTokenizer(PreTrainedTokenizer):
 
     def __init__(
         self,
-        add_bos_token=True,
         add_eos_token=True,
-        add_dim_token=True,
+        add_dim_token=False,
         clean_up_tokenization_spaces=False,
         use_default_system_prompt=False,
         **kwargs,
     ):
         self._tokenizer = AbsTokenizer()
 
-        self.add_bos_token = add_bos_token
         self.add_eos_token = add_eos_token
         self.add_dim_token = add_dim_token
         self.use_default_system_prompt = use_default_system_prompt
@@ -77,35 +75,35 @@ class AriaTokenizer(PreTrainedTokenizer):
     def tokenize(
         self,
         midi_dict: MidiDict,
-        add_dim_tok: Optional[bool] = None,
-        add_eos_tok: Optional[bool] = None,
+        add_dim_token: Optional[bool] = None,
+        add_eos_token: Optional[bool] = None,
         **kwargs,
     ) -> List[Token]:
         return self._tokenizer.tokenize(
             midi_dict=midi_dict,
             add_dim_tok=(
-                add_dim_tok if add_dim_tok is not None else self.add_dim_token
+                add_dim_token
+                if add_dim_token is not None
+                else self.add_dim_token
             ),
             add_eos_tok=(
-                add_eos_tok if add_eos_tok is not None else self.add_eos_token
+                add_eos_token
+                if add_eos_token is not None
+                else self.add_eos_token
             ),
         )
 
     def _tokenize(
         self,
         midi_dict: MidiDict,
-        add_dim_tok: Optional[bool] = None,
-        add_eos_tok: Optional[bool] = None,
+        add_dim_token: Optional[bool] = None,
+        add_eos_token: Optional[bool] = None,
         **kwargs,
     ) -> List[Token]:
         return self._tokenizer.tokenize(
             midi_dict=midi_dict,
-            add_dim_tok=(
-                add_dim_tok if add_dim_tok is not None else self.add_dim_token
-            ),
-            add_eos_tok=(
-                add_eos_tok if add_eos_tok is not None else self.add_eos_token
-            ),
+            add_dim_tok=add_dim_token,
+            add_eos_tok=add_eos_token,
         )
 
     def __call__(

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
-  "add_bos_token": false,
-  "add_eos_token": false,
+  "add_eos_token": true,
+  "add_dim_token": false,
   "auto_map": {
     "AutoTokenizer": [
       "tokenization_aria.AriaTokenizer",