| """ | |
| Derived from Andrej Karpathy's nanochat project. | |
| MIT License | |
| Copyright (c) 2025 Andrej Karpathy | |
| Permission is hereby granted, free of charge, to any person obtaining a copy | |
| of this software and associated documentation files (the "Software"), to deal | |
| in the Software without restriction, including without limitation the rights | |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
| copies of the Software, and to permit persons to whom the Software is | |
| furnished to do so, subject to the following conditions: | |
| The above copyright notice and this permission notice shall be included in all | |
| copies or substantial portions of the Software. | |
| """ | |
| from dropout_decay.tokenization.bpe import BpeTokenizer, SPECIAL_TOKENS, SPLIT_PATTERN | |
| BytePairTokenizer = BpeTokenizer | |
| __all__ = [ | |
| "BpeTokenizer", | |
| "BytePairTokenizer", | |
| "SPECIAL_TOKENS", | |
| "SPLIT_PATTERN", | |
| ] | |