Upload processor
Browse files- processing_action_tokenizer.py +3 -4
- tokenizer.json +0 -0
processing_action_tokenizer.py
CHANGED
|
@@ -134,14 +134,14 @@ class UniversalActionProcessor(ProcessorMixin):
|
|
| 134 |
bpe = ByteLevelBPETokenizer()
|
| 135 |
|
| 136 |
# Set up the entire range of possible tokens as the initial alphabet
|
| 137 |
-
|
| 138 |
trainer = BpeTrainer(
|
| 139 |
vocab_size=vocab_size,
|
| 140 |
min_frequency=2,
|
| 141 |
show_progress=True,
|
| 142 |
special_tokens=[],
|
| 143 |
-
|
| 144 |
-
max_token_length=
|
| 145 |
)
|
| 146 |
|
| 147 |
# Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
|
|
@@ -154,7 +154,6 @@ class UniversalActionProcessor(ProcessorMixin):
|
|
| 154 |
# min_frequency=2,
|
| 155 |
# special_tokens=[],
|
| 156 |
# length=len(dct_tokens),
|
| 157 |
-
# max_token_length=512,
|
| 158 |
# )
|
| 159 |
|
| 160 |
return cls(
|
|
|
|
| 134 |
bpe = ByteLevelBPETokenizer()
|
| 135 |
|
| 136 |
# Set up the entire range of possible tokens as the initial alphabet
|
| 137 |
+
alphabet = [chr(i) for i in range(max_token - min_token + 1)]
|
| 138 |
trainer = BpeTrainer(
|
| 139 |
vocab_size=vocab_size,
|
| 140 |
min_frequency=2,
|
| 141 |
show_progress=True,
|
| 142 |
special_tokens=[],
|
| 143 |
+
initial_alphabet=alphabet,
|
| 144 |
+
max_token_length=256,
|
| 145 |
)
|
| 146 |
|
| 147 |
# Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
|
|
|
|
| 154 |
# min_frequency=2,
|
| 155 |
# special_tokens=[],
|
| 156 |
# length=len(dct_tokens),
|
|
|
|
| 157 |
# )
|
| 158 |
|
| 159 |
return cls(
|
tokenizer.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|