| # Copyright (c) Facebook, Inc. and its affiliates. | |
| # | |
| # This source code is licensed under the MIT license found in the | |
| # LICENSE file in the root directory of this source tree. | |
| import importlib | |
| import os | |
| from fairseq import registry | |
| build_tokenizer, register_tokenizer, TOKENIZER_REGISTRY, _ = registry.setup_registry( | |
| "--tokenizer", | |
| default=None, | |
| ) | |
| build_bpe, register_bpe, BPE_REGISTRY, _ = registry.setup_registry( | |
| "--bpe", | |
| default=None, | |
| ) | |
| # automatically import any Python files in the encoders/ directory | |
| for file in sorted(os.listdir(os.path.dirname(__file__))): | |
| if file.endswith(".py") and not file.startswith("_"): | |
| module = file[: file.find(".py")] | |
| importlib.import_module("fairseq.data.encoders." + module) | |