Add source batch 10/11

.gitattributes

@@ -249,3 +249,21 @@ source/rpds/rpds.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -
 source/safetensors/_safetensors_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
 source/sentencepiece/_sentencepiece.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
 source/tiktoken/_tiktoken.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+source/tokenizers/tokenizers.abi3.so filter=lfs diff=lfs merge=lfs -text
+source/torchaudio/lib/_torchaudio.so filter=lfs diff=lfs merge=lfs -text
+source/torchaudio/lib/libctc_prefix_decoder.so filter=lfs diff=lfs merge=lfs -text
+source/torchaudio/lib/libtorchaudio.so filter=lfs diff=lfs merge=lfs -text
+source/torchaudio/lib/pybind11_prefixctc.so filter=lfs diff=lfs merge=lfs -text
+source/torchvision/_C.so filter=lfs diff=lfs merge=lfs -text
+source/torchvision/image.so filter=lfs diff=lfs merge=lfs -text
+source/torchvision.libs/libcudart.e8e8b82a.so.12 filter=lfs diff=lfs merge=lfs -text
+source/torchvision.libs/libjpeg.d246b9ea.so.8 filter=lfs diff=lfs merge=lfs -text
+source/torchvision.libs/libnvjpeg.8dd2b5e6.so.12 filter=lfs diff=lfs merge=lfs -text
+source/torchvision.libs/libpng16.4ef4b109.so.16 filter=lfs diff=lfs merge=lfs -text
+source/torchvision.libs/libwebp.121d56b5.so.7 filter=lfs diff=lfs merge=lfs -text
+source/torchvision.libs/libz.cac6d5fc.so.1 filter=lfs diff=lfs merge=lfs -text
+source/tvm_ffi/core.abi3.so filter=lfs diff=lfs merge=lfs -text
+source/tvm_ffi/lib/libtvm_ffi.so filter=lfs diff=lfs merge=lfs -text
+source/tvm_ffi/lib/libtvm_ffi_testing.so filter=lfs diff=lfs merge=lfs -text
+source/uvloop/loop.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+source/watchfiles/_rust_notify.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

source/tiktoken-0.12.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/tiktoken-0.12.0.dist-info/METADATA

@@ -0,0 +1,171 @@
+Metadata-Version: 2.4
+Name: tiktoken
+Version: 0.12.0
+Summary: tiktoken is a fast BPE tokeniser for use with OpenAI's models
+Author: Shantanu Jain
+Author-email: shantanu@openai.com
+License: MIT License
+        
+        Copyright (c) 2022 OpenAI, Shantanu Jain
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://github.com/openai/tiktoken
+Project-URL: repository, https://github.com/openai/tiktoken
+Project-URL: changelog, https://github.com/openai/tiktoken/blob/main/CHANGELOG.md
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: regex>=2022.1.18
+Requires-Dist: requests>=2.26.0
+Provides-Extra: blobfile
+Requires-Dist: blobfile>=2; extra == "blobfile"
+Dynamic: license-file
+
+# ⏳ tiktoken
+
+tiktoken is a fast [BPE](https://en.wikipedia.org/wiki/Byte_pair_encoding) tokeniser for use with
+OpenAI's models.
+
+```python
+import tiktoken
+enc = tiktoken.get_encoding("o200k_base")
+assert enc.decode(enc.encode("hello world")) == "hello world"
+
+# To get the tokeniser corresponding to a specific model in the OpenAI API:
+enc = tiktoken.encoding_for_model("gpt-4o")
+```
+
+The open source version of `tiktoken` can be installed from [PyPI](https://pypi.org/project/tiktoken):
+```
+pip install tiktoken
+```
+
+The tokeniser API is documented in `tiktoken/core.py`.
+
+Example code using `tiktoken` can be found in the
+[OpenAI Cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb).
+
+
+## Performance
+
+`tiktoken` is between 3-6x faster than a comparable open source tokeniser:
+
+![image](https://raw.githubusercontent.com/openai/tiktoken/main/perf.svg)
+
+Performance measured on 1GB of text using the GPT-2 tokeniser, using `GPT2TokenizerFast` from
+`tokenizers==0.13.2`, `transformers==4.24.0` and `tiktoken==0.2.0`.
+
+
+## Getting help
+
+Please post questions in the [issue tracker](https://github.com/openai/tiktoken/issues).
+
+If you work at OpenAI, make sure to check the internal documentation or feel free to contact
+@shantanu.
+
+## What is BPE anyway?
+
+Language models don't see text like you and I, instead they see a sequence of numbers (known as tokens).
+Byte pair encoding (BPE) is a way of converting text into tokens. It has a couple desirable
+properties:
+1) It's reversible and lossless, so you can convert tokens back into the original text
+2) It works on arbitrary text, even text that is not in the tokeniser's training data
+3) It compresses the text: the token sequence is shorter than the bytes corresponding to the
+   original text. On average, in practice, each token corresponds to about 4 bytes.
+4) It attempts to let the model see common subwords. For instance, "ing" is a common subword in
+   English, so BPE encodings will often split "encoding" into tokens like "encod" and "ing"
+   (instead of e.g. "enc" and "oding"). Because the model will then see the "ing" token again and
+   again in different contexts, it helps models generalise and better understand grammar.
+
+`tiktoken` contains an educational submodule that is friendlier if you want to learn more about
+the details of BPE, including code that helps visualise the BPE procedure:
+```python
+from tiktoken._educational import *
+
+# Train a BPE tokeniser on a small amount of text
+enc = train_simple_encoding()
+
+# Visualise how the GPT-4 encoder encodes text
+enc = SimpleBytePairEncoding.from_tiktoken("cl100k_base")
+enc.encode("hello world aaaaaaaaaaaa")
+```
+
+
+## Extending tiktoken
+
+You may wish to extend `tiktoken` to support new encodings. There are two ways to do this.
+
+
+**Create your `Encoding` object exactly the way you want and simply pass it around.**
+
+```python
+cl100k_base = tiktoken.get_encoding("cl100k_base")
+
+# In production, load the arguments directly instead of accessing private attributes
+# See openai_public.py for examples of arguments for specific encodings
+enc = tiktoken.Encoding(
+    # If you're changing the set of special tokens, make sure to use a different name
+    # It should be clear from the name what behaviour to expect.
+    name="cl100k_im",
+    pat_str=cl100k_base._pat_str,
+    mergeable_ranks=cl100k_base._mergeable_ranks,
+    special_tokens={
+        **cl100k_base._special_tokens,
+        "<|im_start|>": 100264,
+        "<|im_end|>": 100265,
+    }
+)
+```
+
+**Use the `tiktoken_ext` plugin mechanism to register your `Encoding` objects with `tiktoken`.**
+
+This is only useful if you need `tiktoken.get_encoding` to find your encoding, otherwise prefer
+option 1.
+
+To do this, you'll need to create a namespace package under `tiktoken_ext`.
+
+Layout your project like this, making sure to omit the `tiktoken_ext/__init__.py` file:
+```
+my_tiktoken_extension
+├── tiktoken_ext
+│   └── my_encodings.py
+└── setup.py
+```
+
+`my_encodings.py` should be a module that contains a variable named `ENCODING_CONSTRUCTORS`.
+This is a dictionary from an encoding name to a function that takes no arguments and returns
+arguments that can be passed to `tiktoken.Encoding` to construct that encoding. For an example, see
+`tiktoken_ext/openai_public.py`. For precise details, see `tiktoken/registry.py`.
+
+Your `setup.py` should look something like this:
+```python
+from setuptools import setup, find_namespace_packages
+
+setup(
+    name="my_tiktoken_extension",
+    packages=find_namespace_packages(include=['tiktoken_ext*']),
+    install_requires=["tiktoken"],
+    ...
+)
+```
+
+Then simply `pip install ./my_tiktoken_extension` and you should be able to use your
+custom encodings! Make sure **not** to use an editable install.
+

source/tiktoken-0.12.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+tiktoken-0.12.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+tiktoken-0.12.0.dist-info/METADATA,sha256=07KObsjwnEfLTQ-oRD0vhrE3Zw-oNMJqTlBbEeJxtZ8,6688
+tiktoken-0.12.0.dist-info/RECORD,,
+tiktoken-0.12.0.dist-info/WHEEL,sha256=VXvNKn6nFeCM45GEUrNLJOO_J_e-cNJphGt9rWFxyE0,113
+tiktoken-0.12.0.dist-info/licenses/LICENSE,sha256=QYy0mbQ2Eo1lPXmUEzOlQ3t74uqSE9zC8E0V1dLFHYY,1078
+tiktoken-0.12.0.dist-info/top_level.txt,sha256=54G5MceQnuD7EXvp7jzGxDDapA1iOwsh77jhCN9WKkc,22
+tiktoken/__init__.py,sha256=eHlkakibO43-11JFQJUgpC8z2v4ID1r3l3LXjMyEwKc,346
+tiktoken/__pycache__/__init__.cpython-312.pyc,,
+tiktoken/__pycache__/_educational.cpython-312.pyc,,
+tiktoken/__pycache__/core.cpython-312.pyc,,
+tiktoken/__pycache__/load.cpython-312.pyc,,
+tiktoken/__pycache__/model.cpython-312.pyc,,
+tiktoken/__pycache__/registry.cpython-312.pyc,,
+tiktoken/_educational.py,sha256=TUFOp8Q91WjrTvGKhCNEyrhtva82UlenXfhPy9zS7VQ,8229
+tiktoken/_tiktoken.cpython-312-x86_64-linux-gnu.so,sha256=qCn0iO_VQ7YJKD6D8jDXz6WHH64mSUK59dicngi37S8,3525056
+tiktoken/core.py,sha256=TCwORlettZl-da55Ysp52TlLk18nKD6e62Q_0ZFA404,17458
+tiktoken/load.py,sha256=dhTOiVIInbhiQ_zmtOZDshKvqSKzXyNOJJWPmJ0S9RU,5919
+tiktoken/model.py,sha256=d57kixsksIv6VESndVjvmGBRj8LrSFGAwUCV5xZtxRk,4061
+tiktoken/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tiktoken/registry.py,sha256=7fktZbJ1Kcm8sVyWgEfIy-ZxfUvcXupLUNXKPfSGwQU,3256
+tiktoken_ext/__pycache__/openai_public.cpython-312.pyc,,
+tiktoken_ext/openai_public.py,sha256=lUOSc45g0Pttyh2tgIcu_EfI4nM7q-y78KI5cO1mwss,5613

source/tiktoken-0.12.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: false
+Tag: cp312-cp312-manylinux_2_28_x86_64
+

source/tiktoken-0.12.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 OpenAI, Shantanu Jain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

source/tiktoken-0.12.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+tiktoken
+tiktoken_ext

source/tiktoken_ext/openai_public.py

@@ -0,0 +1,162 @@
+from tiktoken.load import data_gym_to_mergeable_bpe_ranks, load_tiktoken_bpe
+
+ENDOFTEXT = "<|endoftext|>"
+FIM_PREFIX = "<|fim_prefix|>"
+FIM_MIDDLE = "<|fim_middle|>"
+FIM_SUFFIX = "<|fim_suffix|>"
+ENDOFPROMPT = "<|endofprompt|>"
+
+# The pattern in the original GPT-2 release is:
+# r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+# This is equivalent, but executes faster:
+r50k_pat_str = (
+    r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"""
+)
+
+
+def gpt2():
+    mergeable_ranks = data_gym_to_mergeable_bpe_ranks(
+        vocab_bpe_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpe",
+        encoder_json_file="https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json",
+        vocab_bpe_hash="1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5",
+        encoder_json_hash="196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783",
+    )
+    return {
+        "name": "gpt2",
+        "explicit_n_vocab": 50257,
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {ENDOFTEXT: 50256},
+    }
+
+
+def r50k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
+        expected_hash="306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
+    )
+    return {
+        "name": "r50k_base",
+        "explicit_n_vocab": 50257,
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {ENDOFTEXT: 50256},
+    }
+
+
+def p50k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+        expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069",
+    )
+    return {
+        "name": "p50k_base",
+        "explicit_n_vocab": 50281,
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {ENDOFTEXT: 50256},
+    }
+
+
+def p50k_edit():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+        expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069",
+    )
+    special_tokens = {ENDOFTEXT: 50256, FIM_PREFIX: 50281, FIM_MIDDLE: 50282, FIM_SUFFIX: 50283}
+    return {
+        "name": "p50k_edit",
+        "pat_str": r50k_pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+def cl100k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
+        expected_hash="223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7",
+    )
+    special_tokens = {
+        ENDOFTEXT: 100257,
+        FIM_PREFIX: 100258,
+        FIM_MIDDLE: 100259,
+        FIM_SUFFIX: 100260,
+        ENDOFPROMPT: 100276,
+    }
+    return {
+        "name": "cl100k_base",
+        "pat_str": r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s""",
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+def o200k_base():
+    mergeable_ranks = load_tiktoken_bpe(
+        "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+        expected_hash="446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d",
+    )
+    special_tokens = {ENDOFTEXT: 199999, ENDOFPROMPT: 200018}
+    # This regex could be made more efficient. If I was the one working on this encoding, I would
+    # have done a few other things differently too, e.g. I think you can allocate tokens more
+    # efficiently across languages.
+    pat_str = "|".join(
+        [
+            r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
+            r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
+            r"""\p{N}{1,3}""",
+            r""" ?[^\s\p{L}\p{N}]+[\r\n/]*""",
+            r"""\s*[\r\n]+""",
+            r"""\s+(?!\S)""",
+            r"""\s+""",
+        ]
+    )
+    return {
+        "name": "o200k_base",
+        "pat_str": pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+def o200k_harmony():
+    base_enc = o200k_base()
+    name = "o200k_harmony"
+    pat_str = base_enc["pat_str"]
+    mergeable_ranks = base_enc["mergeable_ranks"]
+    special_tokens = {
+        **base_enc["special_tokens"],
+        "<|startoftext|>": 199998,
+        "<|endoftext|>": 199999,
+        "<|reserved_200000|>": 200000,
+        "<|reserved_200001|>": 200001,
+        "<|return|>": 200002,
+        "<|constrain|>": 200003,
+        "<|reserved_200004|>": 200004,
+        "<|channel|>": 200005,
+        "<|start|>": 200006,
+        "<|end|>": 200007,
+        "<|message|>": 200008,
+        "<|reserved_200009|>": 200009,
+        "<|reserved_200010|>": 200010,
+        "<|reserved_200011|>": 200011,
+        "<|call|>": 200012,
+    } | {f"<|reserved_{i}|>": i for i in range(200013, 201088)}
+    return {
+        "name": name,
+        "pat_str": pat_str,
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": special_tokens,
+    }
+
+
+ENCODING_CONSTRUCTORS = {
+    "gpt2": gpt2,
+    "r50k_base": r50k_base,
+    "p50k_base": p50k_base,
+    "p50k_edit": p50k_edit,
+    "cl100k_base": cl100k_base,
+    "o200k_base": o200k_base,
+    "o200k_harmony": o200k_harmony,
+}

source/tokenizers-0.22.2.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/tokenizers-0.22.2.dist-info/METADATA

@@ -0,0 +1,214 @@
+Metadata-Version: 2.4
+Name: tokenizers
+Version: 0.22.2
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: huggingface-hub>=0.16.4,<2.0
+Requires-Dist: pytest ; extra == 'testing'
+Requires-Dist: pytest-asyncio ; extra == 'testing'
+Requires-Dist: requests ; extra == 'testing'
+Requires-Dist: numpy ; extra == 'testing'
+Requires-Dist: datasets ; extra == 'testing'
+Requires-Dist: ruff ; extra == 'testing'
+Requires-Dist: ty ; extra == 'testing'
+Requires-Dist: sphinx ; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
+Requires-Dist: setuptools-rust ; extra == 'docs'
+Requires-Dist: tokenizers[testing] ; extra == 'dev'
+Provides-Extra: testing
+Provides-Extra: docs
+Provides-Extra: dev
+Keywords: NLP,tokenizer,BPE,transformer,deep learning
+Author-email: Nicolas Patry <patry.nicolas@protonmail.com>, Anthony Moi <anthony@huggingface.co>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/huggingface/tokenizers
+Project-URL: Source, https://github.com/huggingface/tokenizers
+
+<p align="center">
+    <br>
+    <img src="https://huggingface.co/landing/assets/tokenizers/tokenizers-logo.png" width="600"/>
+    <br>
+<p>
+<p align="center">
+    <a href="https://badge.fury.io/py/tokenizers">
+         <img alt="Build" src="https://badge.fury.io/py/tokenizers.svg">
+    </a>
+    <a href="https://github.com/huggingface/tokenizers/blob/master/LICENSE">
+        <img alt="GitHub" src="https://img.shields.io/github/license/huggingface/tokenizers.svg?color=blue">
+    </a>
+</p>
+<br>
+
+# Tokenizers
+
+Provides an implementation of today's most used tokenizers, with a focus on performance and
+versatility.
+
+Bindings over the [Rust](https://github.com/huggingface/tokenizers/tree/master/tokenizers) implementation.
+If you are interested in the High-level design, you can go check it there.
+
+Otherwise, let's dive in!
+
+## Main features:
+
+ - Train new vocabularies and tokenize using 4 pre-made tokenizers (Bert WordPiece and the 3
+   most common BPE versions).
+ - Extremely fast (both training and tokenization), thanks to the Rust implementation. Takes
+   less than 20 seconds to tokenize a GB of text on a server's CPU.
+ - Easy to use, but also extremely versatile.
+ - Designed for research and production.
+ - Normalization comes with alignments tracking. It's always possible to get the part of the
+   original sentence that corresponds to a given token.
+ - Does all the pre-processing: Truncate, Pad, add the special tokens your model needs.
+
+### Installation
+
+#### With pip:
+
+```bash
+pip install tokenizers
+```
+
+#### From sources:
+
+To use this method, you need to have the Rust installed:
+
+```bash
+# Install with:
+curl https://sh.rustup.rs -sSf | sh -s -- -y
+export PATH="$HOME/.cargo/bin:$PATH"
+```
+
+Once Rust is installed, you can compile doing the following
+
+```bash
+git clone https://github.com/huggingface/tokenizers
+cd tokenizers/bindings/python
+
+# Create a virtual env (you can use yours as well)
+python -m venv .env
+source .env/bin/activate
+
+# Install `tokenizers` in the current virtual env
+pip install -e .
+```
+
+### Load a pretrained tokenizer from the Hub
+
+```python
+from tokenizers import Tokenizer
+
+tokenizer = Tokenizer.from_pretrained("bert-base-cased")
+```
+
+### Using the provided Tokenizers
+
+We provide some pre-build tokenizers to cover the most common cases. You can easily load one of
+these using some `vocab.json` and `merges.txt` files:
+
+```python
+from tokenizers import CharBPETokenizer
+
+# Initialize a tokenizer
+vocab = "./path/to/vocab.json"
+merges = "./path/to/merges.txt"
+tokenizer = CharBPETokenizer(vocab, merges)
+
+# And then encode:
+encoded = tokenizer.encode("I can feel the magic, can you?")
+print(encoded.ids)
+print(encoded.tokens)
+```
+
+And you can train them just as simply:
+
+```python
+from tokenizers import CharBPETokenizer
+
+# Initialize a tokenizer
+tokenizer = CharBPETokenizer()
+
+# Then train it!
+tokenizer.train([ "./path/to/files/1.txt", "./path/to/files/2.txt" ])
+
+# Now, let's use it:
+encoded = tokenizer.encode("I can feel the magic, can you?")
+
+# And finally save it somewhere
+tokenizer.save("./path/to/directory/my-bpe.tokenizer.json")
+```
+
+#### Provided Tokenizers
+
+ - `CharBPETokenizer`: The original BPE
+ - `ByteLevelBPETokenizer`: The byte level version of the BPE
+ - `SentencePieceBPETokenizer`: A BPE implementation compatible with the one used by SentencePiece
+ - `BertWordPieceTokenizer`: The famous Bert tokenizer, using WordPiece
+
+All of these can be used and trained as explained above!
+
+### Build your own
+
+Whenever these provided tokenizers don't give you enough freedom, you can build your own tokenizer,
+by putting all the different parts you need together.
+You can check how we implemented the [provided tokenizers](https://github.com/huggingface/tokenizers/tree/master/bindings/python/py_src/tokenizers/implementations) and adapt them easily to your own needs.
+
+#### Building a byte-level BPE
+
+Here is an example showing how to build your own byte-level BPE by putting all the different pieces
+together, and then saving it to a single file:
+
+```python
+from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers, processors
+
+# Initialize a tokenizer
+tokenizer = Tokenizer(models.BPE())
+
+# Customize pre-tokenization and decoding
+tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True)
+tokenizer.decoder = decoders.ByteLevel()
+tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
+
+# And then train
+trainer = trainers.BpeTrainer(
+    vocab_size=20000,
+    min_frequency=2,
+    initial_alphabet=pre_tokenizers.ByteLevel.alphabet()
+)
+tokenizer.train([
+    "./path/to/dataset/1.txt",
+    "./path/to/dataset/2.txt",
+    "./path/to/dataset/3.txt"
+], trainer=trainer)
+
+# And Save it
+tokenizer.save("byte-level-bpe.tokenizer.json", pretty=True)
+```
+
+Now, when you want to use this tokenizer, this is as simple as:
+
+```python
+from tokenizers import Tokenizer
+
+tokenizer = Tokenizer.from_file("byte-level-bpe.tokenizer.json")
+
+encoded = tokenizer.encode("I can feel the magic, can you?")
+```
+
+### Typing support and `stub.py`
+
+The compiled PyO3 extension does not expose type annotations, so editors and type checkers would otherwise see most objects as `Any`. The `stub.py` helper walks the loaded extension modules, renders `.pyi` stub files (plus minimal forwarding `__init__.py` shims), and formats them so that tools like mypy/pyright can understand the public API. Run `python stub.py` whenever you change the Python-visible surface to keep the generated stubs in sync.
+

source/tokenizers-0.22.2.dist-info/RECORD

@@ -0,0 +1,46 @@
+tokenizers-0.22.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+tokenizers-0.22.2.dist-info/METADATA,sha256=FaXdr0ifWSt34Kk0wO60a1ETCpQGTKEpIyr9sKOGjvw,7254
+tokenizers-0.22.2.dist-info/RECORD,,
+tokenizers-0.22.2.dist-info/WHEEL,sha256=5mwg5nCvp3YrLxikUrE5E0HBDKerMOoBBb70NjCncME,143
+tokenizers/__init__.py,sha256=FI7LEi8_7gO-mrsf4hPdhfvGkb8q0rQ3_1MVM3gaajo,2639
+tokenizers/__init__.pyi,sha256=MKWF2m4mz7IG1bPTdJ7AjXkQDNzkmQSLMmACQ2VUYJU,55891
+tokenizers/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/decoders/__init__.py,sha256=hfwM6CFUDvlMGGL4-xsaaYz81K9P5rQI5ZL5UHWK8Y4,372
+tokenizers/decoders/__init__.pyi,sha256=T60mFckMbS8YrsonOAPtfvb7VYHUJi9mm47Wd8pT62o,12019
+tokenizers/decoders/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/implementations/__init__.py,sha256=VzAsplaIo7rl4AFO8Miu7ig7MfZjvonwVblZw01zR6M,310
+tokenizers/implementations/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/implementations/__pycache__/base_tokenizer.cpython-312.pyc,,
+tokenizers/implementations/__pycache__/bert_wordpiece.cpython-312.pyc,,
+tokenizers/implementations/__pycache__/byte_level_bpe.cpython-312.pyc,,
+tokenizers/implementations/__pycache__/char_level_bpe.cpython-312.pyc,,
+tokenizers/implementations/__pycache__/sentencepiece_bpe.cpython-312.pyc,,
+tokenizers/implementations/__pycache__/sentencepiece_unigram.cpython-312.pyc,,
+tokenizers/implementations/base_tokenizer.py,sha256=PtQ2TSmoMGlTpL8oc8fDvwJVIY6isWGmps9comzsWjE,15806
+tokenizers/implementations/bert_wordpiece.py,sha256=sKCum0FKPYdSgJFJN8LDerVBoTDRSqyqSdrcm-lvQqI,5520
+tokenizers/implementations/byte_level_bpe.py,sha256=iBepM_z1s5Ky7zFDVrYLc3L5byYrIouk7-k0JGuF10s,4272
+tokenizers/implementations/char_level_bpe.py,sha256=Nag_HFq8Rvcucqi8MhV1-0xtoR0C7FjHOecFVURL7ss,5449
+tokenizers/implementations/sentencepiece_bpe.py,sha256=c08fKf6i92E2RsKgsxy7LzZfYX8-MACHSRG8U_I5ytY,3721
+tokenizers/implementations/sentencepiece_unigram.py,sha256=2RoIfFVpiMkJOtOCskM_VCeCELWaC_bNnds6GvtE0KQ,7630
+tokenizers/models/__init__.py,sha256=eJZ4HTAQZpxnKILNylWaTFqxXy-Ba6OKswWN47feeV8,176
+tokenizers/models/__init__.pyi,sha256=2gZPQR1Z5_krTzLXx-ts5ai7Fz7bTZ0QI1OSJ5MyOuc,19517
+tokenizers/models/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/normalizers/__init__.py,sha256=_06w4cqRItveEgIddYaLMScgkSOkIAMIzYCesb5AA4U,841
+tokenizers/normalizers/__init__.pyi,sha256=6zYmbFtvdF1WhoWQSdEN974mxHjc7ZwJBA0TI2dJk98,25709
+tokenizers/normalizers/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/pre_tokenizers/__init__.py,sha256=KV9-EsAykGENUUzkGWCbv4n6YM6hYa1hfnY-gzBpMNE,598
+tokenizers/pre_tokenizers/__init__.pyi,sha256=_pc34-Kd2N7Nvs7vTHPULBKjm18iJRM9qLOClVHw9n4,31566
+tokenizers/pre_tokenizers/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/processors/__init__.py,sha256=xM2DEKwKtHIumHsszM8AMkq-AlaqvBZFXWgLU8SNhOY,307
+tokenizers/processors/__init__.pyi,sha256=5L5OBZ7SXCg7AEy51jyDHViaCSHG5c7vW4eWjSVQbUs,14348
+tokenizers/processors/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/tokenizers.abi3.so,sha256=wRb88egNRhzgo1wzKXTyWUnoNZQW9Qs9UzcYENLOHMw,10074176
+tokenizers/tokenizers.pyi,sha256=Mq4G5RcxKiVc0FZd_Omi-bT7YQMRc-iDBU_nPCmCZOA,468
+tokenizers/tools/__init__.py,sha256=xG8caB9OHC8cbB01S5vYV14HZxhO6eWbLehsb70ppio,55
+tokenizers/tools/__pycache__/__init__.cpython-312.pyc,,
+tokenizers/tools/__pycache__/visualizer.cpython-312.pyc,,
+tokenizers/tools/visualizer-styles.css,sha256=zAydq1oGWD8QEll4-eyL8Llw0B1sty_hpIE3tYxL02k,4850
+tokenizers/tools/visualizer.py,sha256=jtxka01phNP47uQSocIQFO_DMnL3ZHdwohGVDqqYJPo,14834
+tokenizers/trainers/__init__.py,sha256=UTu22AGcp76IvpW45xLRbJWET04NxPW6NfCb2YYz0EM,248
+tokenizers/trainers/__init__.pyi,sha256=jKtDNXnoX6FWeCTeHz-W62Cj2_JErgYG7h1PReUz1rU,10719
+tokenizers/trainers/__pycache__/__init__.cpython-312.pyc,,

source/tokenizers-0.22.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: maturin (1.10.2)
+Root-Is-Purelib: false
+Tag: cp39-abi3-manylinux_2_17_x86_64
+Tag: cp39-abi3-manylinux2014_x86_64

source/tokenizers/__init__.py

@@ -0,0 +1,100 @@
+from enum import Enum
+from typing import List, Tuple, Union
+
+
+Offsets = Tuple[int, int]
+
+TextInputSequence = str
+"""A :obj:`str` that represents an input sequence """
+
+PreTokenizedInputSequence = Union[List[str], Tuple[str]]
+"""A pre-tokenized input sequence. Can be one of:
+
+    - A :obj:`List` of :obj:`str`
+    - A :obj:`Tuple` of :obj:`str`
+"""
+
+TextEncodeInput = Union[
+    TextInputSequence,
+    Tuple[TextInputSequence, TextInputSequence],
+    List[TextInputSequence],
+]
+"""Represents a textual input for encoding. Can be either:
+
+    - A single sequence: :data:`~tokenizers.TextInputSequence`
+    - A pair of sequences:
+
+      - A :obj:`Tuple` of :data:`~tokenizers.TextInputSequence`
+      - Or a :obj:`List` of :data:`~tokenizers.TextInputSequence` of size 2
+"""
+
+PreTokenizedEncodeInput = Union[
+    PreTokenizedInputSequence,
+    Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
+    List[PreTokenizedInputSequence],
+]
+"""Represents a pre-tokenized input for encoding. Can be either:
+
+    - A single sequence: :data:`~tokenizers.PreTokenizedInputSequence`
+    - A pair of sequences:
+
+      - A :obj:`Tuple` of :data:`~tokenizers.PreTokenizedInputSequence`
+      - Or a :obj:`List` of :data:`~tokenizers.PreTokenizedInputSequence` of size 2
+"""
+
+InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
+"""Represents all the possible types of input sequences for encoding. Can be:
+
+    - When ``is_pretokenized=False``: :data:`~TextInputSequence`
+    - When ``is_pretokenized=True``: :data:`~PreTokenizedInputSequence`
+"""
+
+EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
+"""Represents all the possible types of input for encoding. Can be:
+
+    - When ``is_pretokenized=False``: :data:`~TextEncodeInput`
+    - When ``is_pretokenized=True``: :data:`~PreTokenizedEncodeInput`
+"""
+
+
+class OffsetReferential(Enum):
+    ORIGINAL = "original"
+    NORMALIZED = "normalized"
+
+
+class OffsetType(Enum):
+    BYTE = "byte"
+    CHAR = "char"
+
+
+class SplitDelimiterBehavior(Enum):
+    REMOVED = "removed"
+    ISOLATED = "isolated"
+    MERGED_WITH_PREVIOUS = "merged_with_previous"
+    MERGED_WITH_NEXT = "merged_with_next"
+    CONTIGUOUS = "contiguous"
+
+
+from .tokenizers import (  # type: ignore[import]
+    AddedToken,
+    Encoding,
+    NormalizedString,
+    PreTokenizedString,
+    Regex,
+    Token,
+    Tokenizer,
+    decoders,
+    models,
+    normalizers,
+    pre_tokenizers,
+    processors,
+    trainers,
+    __version__,
+)
+from .implementations import (
+    BertWordPieceTokenizer,
+    ByteLevelBPETokenizer,
+    CharBPETokenizer,
+    SentencePieceBPETokenizer,
+    SentencePieceUnigramTokenizer,
+)

source/tokenizers/__init__.pyi

@@ -0,0 +1,1800 @@
+# Generated content DO NOT EDIT
+class AddedToken:
+    """
+    Represents a token that can be be added to a :class:`~tokenizers.Tokenizer`.
+    It can have special options that defines the way it should behave.
+
+    Args:
+        content (:obj:`str`): The content of the token
+
+        single_word (:obj:`bool`, defaults to :obj:`False`):
+            Defines whether this token should only match single words. If :obj:`True`, this
+            token will never match inside of a word. For example the token ``ing`` would match
+            on ``tokenizing`` if this option is :obj:`False`, but not if it is :obj:`True`.
+            The notion of "`inside of a word`" is defined by the word boundaries pattern in
+            regular expressions (ie. the token should start and end with word boundaries).
+
+        lstrip (:obj:`bool`, defaults to :obj:`False`):
+            Defines whether this token should strip all potential whitespaces on its left side.
+            If :obj:`True`, this token will greedily match any whitespace on its left. For
+            example if we try to match the token ``[MASK]`` with ``lstrip=True``, in the text
+            ``"I saw a [MASK]"``, we would match on ``" [MASK]"``. (Note the space on the left).
+
+        rstrip (:obj:`bool`, defaults to :obj:`False`):
+            Defines whether this token should strip all potential whitespaces on its right
+            side. If :obj:`True`, this token will greedily match any whitespace on its right.
+            It works just like :obj:`lstrip` but on the right.
+
+        normalized (:obj:`bool`, defaults to :obj:`True` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
+            Defines whether this token should match against the normalized version of the input
+            text. For example, with the added token ``"yesterday"``, and a normalizer in charge of
+            lowercasing the text, the token could be extract from the input ``"I saw a lion
+            Yesterday"``.
+        special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
+            Defines whether this token should be skipped when decoding.
+
+    """
+    def __init__(self, content=None, single_word=False, lstrip=False, rstrip=False, normalized=True, special=False):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def content(self):
+        """
+        Get the content of this :obj:`AddedToken`
+        """
+        pass
+
+    @content.setter
+    def content(self, value):
+        """
+        Get the content of this :obj:`AddedToken`
+        """
+        pass
+
+    @property
+    def lstrip(self):
+        """
+        Get the value of the :obj:`lstrip` option
+        """
+        pass
+
+    @lstrip.setter
+    def lstrip(self, value):
+        """
+        Get the value of the :obj:`lstrip` option
+        """
+        pass
+
+    @property
+    def normalized(self):
+        """
+        Get the value of the :obj:`normalized` option
+        """
+        pass
+
+    @normalized.setter
+    def normalized(self, value):
+        """
+        Get the value of the :obj:`normalized` option
+        """
+        pass
+
+    @property
+    def rstrip(self):
+        """
+        Get the value of the :obj:`rstrip` option
+        """
+        pass
+
+    @rstrip.setter
+    def rstrip(self, value):
+        """
+        Get the value of the :obj:`rstrip` option
+        """
+        pass
+
+    @property
+    def single_word(self):
+        """
+        Get the value of the :obj:`single_word` option
+        """
+        pass
+
+    @single_word.setter
+    def single_word(self, value):
+        """
+        Get the value of the :obj:`single_word` option
+        """
+        pass
+
+    @property
+    def special(self):
+        """
+        Get the value of the :obj:`special` option
+        """
+        pass
+
+    @special.setter
+    def special(self, value):
+        """
+        Get the value of the :obj:`special` option
+        """
+        pass
+
+class Encoding:
+    """
+    The :class:`~tokenizers.Encoding` represents the output of a :class:`~tokenizers.Tokenizer`.
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def attention_mask(self):
+        """
+        The attention mask
+
+        This indicates to the LM which tokens should be attended to, and which should not.
+        This is especially important when batching sequences, where we need to applying
+        padding.
+
+        Returns:
+           :obj:`List[int]`: The attention mask
+        """
+        pass
+
+    @attention_mask.setter
+    def attention_mask(self, value):
+        """
+        The attention mask
+
+        This indicates to the LM which tokens should be attended to, and which should not.
+        This is especially important when batching sequences, where we need to applying
+        padding.
+
+        Returns:
+           :obj:`List[int]`: The attention mask
+        """
+        pass
+
+    def char_to_token(self, char_pos, sequence_index=0):
+        """
+        Get the token that contains the char at the given position in the input sequence.
+
+        Args:
+            char_pos (:obj:`int`):
+                The position of a char in the input string
+            sequence_index (:obj:`int`, defaults to :obj:`0`):
+                The index of the sequence that contains the target char
+
+        Returns:
+            :obj:`int`: The index of the token that contains this char in the encoded sequence
+        """
+        pass
+
+    def char_to_word(self, char_pos, sequence_index=0):
+        """
+        Get the word that contains the char at the given position in the input sequence.
+
+        Args:
+            char_pos (:obj:`int`):
+                The position of a char in the input string
+            sequence_index (:obj:`int`, defaults to :obj:`0`):
+                The index of the sequence that contains the target char
+
+        Returns:
+            :obj:`int`: The index of the word that contains this char in the input sequence
+        """
+        pass
+
+    @property
+    def ids(self):
+        """
+        The generated IDs
+
+        The IDs are the main input to a Language Model. They are the token indices,
+        the numerical representations that a LM understands.
+
+        Returns:
+            :obj:`List[int]`: The list of IDs
+        """
+        pass
+
+    @ids.setter
+    def ids(self, value):
+        """
+        The generated IDs
+
+        The IDs are the main input to a Language Model. They are the token indices,
+        the numerical representations that a LM understands.
+
+        Returns:
+            :obj:`List[int]`: The list of IDs
+        """
+        pass
+
+    @staticmethod
+    def merge(encodings, growing_offsets=True):
+        """
+        Merge the list of encodings into one final :class:`~tokenizers.Encoding`
+
+        Args:
+            encodings (A :obj:`List` of :class:`~tokenizers.Encoding`):
+                The list of encodings that should be merged in one
+
+            growing_offsets (:obj:`bool`, defaults to :obj:`True`):
+                Whether the offsets should accumulate while merging
+
+        Returns:
+            :class:`~tokenizers.Encoding`: The resulting Encoding
+        """
+        pass
+
+    @property
+    def n_sequences(self):
+        """
+        The number of sequences represented
+
+        Returns:
+            :obj:`int`: The number of sequences in this :class:`~tokenizers.Encoding`
+        """
+        pass
+
+    @n_sequences.setter
+    def n_sequences(self, value):
+        """
+        The number of sequences represented
+
+        Returns:
+            :obj:`int`: The number of sequences in this :class:`~tokenizers.Encoding`
+        """
+        pass
+
+    @property
+    def offsets(self):
+        """
+        The offsets associated to each token
+
+        These offsets let's you slice the input string, and thus retrieve the original
+        part that led to producing the corresponding token.
+
+        Returns:
+            A :obj:`List` of :obj:`Tuple[int, int]`: The list of offsets
+        """
+        pass
+
+    @offsets.setter
+    def offsets(self, value):
+        """
+        The offsets associated to each token
+
+        These offsets let's you slice the input string, and thus retrieve the original
+        part that led to producing the corresponding token.
+
+        Returns:
+            A :obj:`List` of :obj:`Tuple[int, int]`: The list of offsets
+        """
+        pass
+
+    @property
+    def overflowing(self):
+        """
+        A :obj:`List` of overflowing :class:`~tokenizers.Encoding`
+
+        When using truncation, the :class:`~tokenizers.Tokenizer` takes care of splitting
+        the output into as many pieces as required to match the specified maximum length.
+        This field lets you retrieve all the subsequent pieces.
+
+        When you use pairs of sequences, the overflowing pieces will contain enough
+        variations to cover all the possible combinations, while respecting the provided
+        maximum length.
+        """
+        pass
+
+    @overflowing.setter
+    def overflowing(self, value):
+        """
+        A :obj:`List` of overflowing :class:`~tokenizers.Encoding`
+
+        When using truncation, the :class:`~tokenizers.Tokenizer` takes care of splitting
+        the output into as many pieces as required to match the specified maximum length.
+        This field lets you retrieve all the subsequent pieces.
+
+        When you use pairs of sequences, the overflowing pieces will contain enough
+        variations to cover all the possible combinations, while respecting the provided
+        maximum length.
+        """
+        pass
+
+    def pad(self, length, direction="right", pad_id=0, pad_type_id=0, pad_token="[PAD]"):
+        """
+        Pad the :class:`~tokenizers.Encoding` at the given length
+
+        Args:
+            length (:obj:`int`):
+                The desired length
+
+            direction: (:obj:`str`, defaults to :obj:`right`):
+                The expected padding direction. Can be either :obj:`right` or :obj:`left`
+
+            pad_id (:obj:`int`, defaults to :obj:`0`):
+                The ID corresponding to the padding token
+
+            pad_type_id (:obj:`int`, defaults to :obj:`0`):
+                The type ID corresponding to the padding token
+
+            pad_token (:obj:`str`, defaults to `[PAD]`):
+                The pad token to use
+        """
+        pass
+
+    @property
+    def sequence_ids(self):
+        """
+        The generated sequence indices.
+
+        They represent the index of the input sequence associated to each token.
+        The sequence id can be None if the token is not related to any input sequence,
+        like for example with special tokens.
+
+        Returns:
+            A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index.
+        """
+        pass
+
+    @sequence_ids.setter
+    def sequence_ids(self, value):
+        """
+        The generated sequence indices.
+
+        They represent the index of the input sequence associated to each token.
+        The sequence id can be None if the token is not related to any input sequence,
+        like for example with special tokens.
+
+        Returns:
+            A :obj:`List` of :obj:`Optional[int]`: A list of optional sequence index.
+        """
+        pass
+
+    def set_sequence_id(self, sequence_id):
+        """
+        Set the given sequence index
+
+        Set the given sequence index for the whole range of tokens contained in this
+        :class:`~tokenizers.Encoding`.
+        """
+        pass
+
+    @property
+    def special_tokens_mask(self):
+        """
+        The special token mask
+
+        This indicates which tokens are special tokens, and which are not.
+
+        Returns:
+            :obj:`List[int]`: The special tokens mask
+        """
+        pass
+
+    @special_tokens_mask.setter
+    def special_tokens_mask(self, value):
+        """
+        The special token mask
+
+        This indicates which tokens are special tokens, and which are not.
+
+        Returns:
+            :obj:`List[int]`: The special tokens mask
+        """
+        pass
+
+    def token_to_chars(self, token_index):
+        """
+        Get the offsets of the token at the given index.
+
+        The returned offsets are related to the input sequence that contains the
+        token.  In order to determine in which input sequence it belongs, you
+        must call :meth:`~tokenizers.Encoding.token_to_sequence()`.
+
+        Args:
+            token_index (:obj:`int`):
+                The index of a token in the encoded sequence.
+
+        Returns:
+            :obj:`Tuple[int, int]`: The token offsets :obj:`(first, last + 1)`
+        """
+        pass
+
+    def token_to_sequence(self, token_index):
+        """
+        Get the index of the sequence represented by the given token.
+
+        In the general use case, this method returns :obj:`0` for a single sequence or
+        the first sequence of a pair, and :obj:`1` for the second sequence of a pair
+
+        Args:
+            token_index (:obj:`int`):
+                The index of a token in the encoded sequence.
+
+        Returns:
+            :obj:`int`: The sequence id of the given token
+        """
+        pass
+
+    def token_to_word(self, token_index):
+        """
+        Get the index of the word that contains the token in one of the input sequences.
+
+        The returned word index is related to the input sequence that contains
+        the token.  In order to determine in which input sequence it belongs, you
+        must call :meth:`~tokenizers.Encoding.token_to_sequence()`.
+
+        Args:
+            token_index (:obj:`int`):
+                The index of a token in the encoded sequence.
+
+        Returns:
+            :obj:`int`: The index of the word in the relevant input sequence.
+        """
+        pass
+
+    @property
+    def tokens(self):
+        """
+        The generated tokens
+
+        They are the string representation of the IDs.
+
+        Returns:
+            :obj:`List[str]`: The list of tokens
+        """
+        pass
+
+    @tokens.setter
+    def tokens(self, value):
+        """
+        The generated tokens
+
+        They are the string representation of the IDs.
+
+        Returns:
+            :obj:`List[str]`: The list of tokens
+        """
+        pass
+
+    def truncate(self, max_length, stride=0, direction="right"):
+        """
+        Truncate the :class:`~tokenizers.Encoding` at the given length
+
+        If this :class:`~tokenizers.Encoding` represents multiple sequences, when truncating
+        this information is lost. It will be considered as representing a single sequence.
+
+        Args:
+            max_length (:obj:`int`):
+                The desired length
+
+            stride (:obj:`int`, defaults to :obj:`0`):
+                The length of previous content to be included in each overflowing piece
+
+            direction (:obj:`str`, defaults to :obj:`right`):
+                Truncate direction
+        """
+        pass
+
+    @property
+    def type_ids(self):
+        """
+        The generated type IDs
+
+        Generally used for tasks like sequence classification or question answering,
+        these tokens let the LM know which input sequence corresponds to each tokens.
+
+        Returns:
+            :obj:`List[int]`: The list of type ids
+        """
+        pass
+
+    @type_ids.setter
+    def type_ids(self, value):
+        """
+        The generated type IDs
+
+        Generally used for tasks like sequence classification or question answering,
+        these tokens let the LM know which input sequence corresponds to each tokens.
+
+        Returns:
+            :obj:`List[int]`: The list of type ids
+        """
+        pass
+
+    @property
+    def word_ids(self):
+        """
+        The generated word indices.
+
+        They represent the index of the word associated to each token.
+        When the input is pre-tokenized, they correspond to the ID of the given input label,
+        otherwise they correspond to the words indices as defined by the
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
+
+        For special tokens and such (any token that was generated from something that was
+        not part of the input), the output is :obj:`None`
+
+        Returns:
+            A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
+        """
+        pass
+
+    @word_ids.setter
+    def word_ids(self, value):
+        """
+        The generated word indices.
+
+        They represent the index of the word associated to each token.
+        When the input is pre-tokenized, they correspond to the ID of the given input label,
+        otherwise they correspond to the words indices as defined by the
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
+
+        For special tokens and such (any token that was generated from something that was
+        not part of the input), the output is :obj:`None`
+
+        Returns:
+            A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
+        """
+        pass
+
+    def word_to_chars(self, word_index, sequence_index=0):
+        """
+        Get the offsets of the word at the given index in one of the input sequences.
+
+        Args:
+            word_index (:obj:`int`):
+                The index of a word in one of the input sequences.
+            sequence_index (:obj:`int`, defaults to :obj:`0`):
+                The index of the sequence that contains the target word
+
+        Returns:
+            :obj:`Tuple[int, int]`: The range of characters (span) :obj:`(first, last + 1)`
+        """
+        pass
+
+    def word_to_tokens(self, word_index, sequence_index=0):
+        """
+        Get the encoded tokens corresponding to the word at the given index
+        in one of the input sequences.
+
+        Args:
+            word_index (:obj:`int`):
+                The index of a word in one of the input sequences.
+            sequence_index (:obj:`int`, defaults to :obj:`0`):
+                The index of the sequence that contains the target word
+
+        Returns:
+            :obj:`Tuple[int, int]`: The range of tokens: :obj:`(first, last + 1)`
+        """
+        pass
+
+    @property
+    def words(self):
+        """
+        The generated word indices.
+
+        .. warning::
+            This is deprecated and will be removed in a future version.
+            Please use :obj:`~tokenizers.Encoding.word_ids` instead.
+
+        They represent the index of the word associated to each token.
+        When the input is pre-tokenized, they correspond to the ID of the given input label,
+        otherwise they correspond to the words indices as defined by the
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
+
+        For special tokens and such (any token that was generated from something that was
+        not part of the input), the output is :obj:`None`
+
+        Returns:
+            A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
+        """
+        pass
+
+    @words.setter
+    def words(self, value):
+        """
+        The generated word indices.
+
+        .. warning::
+            This is deprecated and will be removed in a future version.
+            Please use :obj:`~tokenizers.Encoding.word_ids` instead.
+
+        They represent the index of the word associated to each token.
+        When the input is pre-tokenized, they correspond to the ID of the given input label,
+        otherwise they correspond to the words indices as defined by the
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` that was used.
+
+        For special tokens and such (any token that was generated from something that was
+        not part of the input), the output is :obj:`None`
+
+        Returns:
+            A :obj:`List` of :obj:`Optional[int]`: A list of optional word index.
+        """
+        pass
+
+class NormalizedString:
+    """
+    NormalizedString
+
+    A NormalizedString takes care of modifying an "original" string, to obtain a "normalized" one.
+    While making all the requested modifications, it keeps track of the alignment information
+    between the two versions of the string.
+
+    Args:
+        sequence: str:
+            The string sequence used to initialize this NormalizedString
+    """
+    def __init__(self, sequence):
+        pass
+
+    def __getitem__(self, key):
+        """
+        Return self[key].
+        """
+        pass
+
+    def __getstate__(self, /):
+        """
+        Helper for pickle.
+        """
+        pass
+
+    def append(self, s):
+        """
+        Append the given sequence to the string
+        """
+        pass
+
+    def clear(self):
+        """
+        Clears the string
+        """
+        pass
+
+    def filter(self, func):
+        """
+        Filter each character of the string using the given func
+        """
+        pass
+
+    def for_each(self, func):
+        """
+        Calls the given function for each character of the string
+        """
+        pass
+
+    def lowercase(self):
+        """
+        Lowercase the string
+        """
+        pass
+
+    def lstrip(self):
+        """
+        Strip the left of the string
+        """
+        pass
+
+    def map(self, func):
+        """
+        Calls the given function for each character of the string
+
+        Replaces each character of the string using the returned value. Each
+        returned value **must** be a str of length 1 (ie a character).
+        """
+        pass
+
+    def nfc(self):
+        """
+        Runs the NFC normalization
+        """
+        pass
+
+    def nfd(self):
+        """
+        Runs the NFD normalization
+        """
+        pass
+
+    def nfkc(self):
+        """
+        Runs the NFKC normalization
+        """
+        pass
+
+    def nfkd(self):
+        """
+        Runs the NFKD normalization
+        """
+        pass
+
+    @property
+    def normalized(self):
+        """
+        The normalized part of the string
+        """
+        pass
+
+    @normalized.setter
+    def normalized(self, value):
+        """
+        The normalized part of the string
+        """
+        pass
+
+    @property
+    def original(self):
+        """ """
+        pass
+
+    @original.setter
+    def original(self, value):
+        """ """
+        pass
+
+    def prepend(self, s):
+        """
+        Prepend the given sequence to the string
+        """
+        pass
+
+    def replace(self, pattern, content):
+        """
+        Replace the content of the given pattern with the provided content
+
+        Args:
+            pattern: Pattern:
+                A pattern used to match the string. Usually a string or a Regex
+
+            content: str:
+                The content to be used as replacement
+        """
+        pass
+
+    def rstrip(self):
+        """
+        Strip the right of the string
+        """
+        pass
+
+    def slice(self, range):
+        """
+        Slice the string using the given range
+        """
+        pass
+
+    def split(self, pattern, behavior):
+        """
+        Split the NormalizedString using the given pattern and the specified behavior
+
+        Args:
+            pattern: Pattern:
+                A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`
+
+            behavior: SplitDelimiterBehavior:
+                The behavior to use when splitting.
+                Choices: "removed", "isolated", "merged_with_previous", "merged_with_next",
+                "contiguous"
+
+        Returns:
+            A list of NormalizedString, representing each split
+        """
+        pass
+
+    def strip(self):
+        """
+        Strip both ends of the string
+        """
+        pass
+
+    def uppercase(self):
+        """
+        Uppercase the string
+        """
+        pass
+
+class PreTokenizedString:
+    """
+    PreTokenizedString
+
+    Wrapper over a string, that provides a way to normalize, pre-tokenize, tokenize the
+    underlying string, while keeping track of the alignment information (offsets).
+
+    The PreTokenizedString manages what we call `splits`. Each split represents a substring
+    which is a subpart of the original string, with the relevant offsets and tokens.
+
+    When calling one of the methods used to modify the PreTokenizedString (namely one of
+    `split`, `normalize` or `tokenize), only the `splits` that don't have any associated
+    tokens will get modified.
+
+    Args:
+        sequence: str:
+            The string sequence used to initialize this PreTokenizedString
+    """
+    def __init__(self, sequence):
+        pass
+
+    def __getstate__(self, /):
+        """
+        Helper for pickle.
+        """
+        pass
+
+    def get_splits(self, offset_referential="original", offset_type="char"):
+        """
+        Get the splits currently managed by the PreTokenizedString
+
+        Args:
+            offset_referential: :obj:`str`
+                Whether the returned splits should have offsets expressed relative
+                to the original string, or the normalized one. choices: "original", "normalized".
+
+            offset_type: :obj:`str`
+                Whether the returned splits should have offsets expressed in bytes or chars.
+                When slicing an str, we usually want to use chars, which is the default value.
+                Now in some cases it might be interesting to get these offsets expressed in bytes,
+                so it is possible to change this here.
+                choices: "char", "bytes"
+
+        Returns
+            A list of splits
+        """
+        pass
+
+    def normalize(self, func):
+        """
+        Normalize each split of the `PreTokenizedString` using the given `func`
+
+        Args:
+            func: Callable[[NormalizedString], None]:
+                The function used to normalize each underlying split. This function
+                does not need to return anything, just calling the methods on the provided
+                NormalizedString allow its modification.
+        """
+        pass
+
+    def split(self, func):
+        """
+        Split the PreTokenizedString using the given `func`
+
+        Args:
+            func: Callable[[index, NormalizedString], List[NormalizedString]]:
+                The function used to split each underlying split.
+                It is expected to return a list of `NormalizedString`, that represent the new
+                splits. If the given `NormalizedString` does not need any splitting, we can
+                just return it directly.
+                In order for the offsets to be tracked accurately, any returned `NormalizedString`
+                should come from calling either `.split` or `.slice` on the received one.
+        """
+        pass
+
+    def to_encoding(self, type_id=0, word_idx=None):
+        """
+        Return an Encoding generated from this PreTokenizedString
+
+        Args:
+            type_id: int = 0:
+                The type_id to be used on the generated Encoding.
+
+            word_idx: Optional[int] = None:
+                An optional word index to be used for each token of this Encoding. If provided,
+                all the word indices in the generated Encoding will use this value, instead
+                of the one automatically tracked during pre-tokenization.
+
+        Returns:
+            An Encoding
+        """
+        pass
+
+    def tokenize(self, func):
+        """
+        Tokenize each split of the `PreTokenizedString` using the given `func`
+
+        Args:
+            func: Callable[[str], List[Token]]:
+                The function used to tokenize each underlying split. This function must return
+                a list of Token generated from the input str.
+        """
+        pass
+
+class Regex:
+    """
+    Instantiate a new Regex with the given pattern
+    """
+    def __init__(self, pattern):
+        pass
+
+    def __getstate__(self, /):
+        """
+        Helper for pickle.
+        """
+        pass
+
+class Token:
+    def __init__(self, id, value, offsets):
+        pass
+
+    def __getstate__(self, /):
+        """
+        Helper for pickle.
+        """
+        pass
+
+    def as_tuple(self):
+        """ """
+        pass
+
+    @property
+    def id(self):
+        """ """
+        pass
+
+    @id.setter
+    def id(self, value):
+        """ """
+        pass
+
+    @property
+    def offsets(self):
+        """ """
+        pass
+
+    @offsets.setter
+    def offsets(self, value):
+        """ """
+        pass
+
+    @property
+    def value(self):
+        """ """
+        pass
+
+    @value.setter
+    def value(self, value):
+        """ """
+        pass
+
+class Tokenizer:
+    """
+    A :obj:`Tokenizer` works as a pipeline. It processes some raw text as input
+    and outputs an :class:`~tokenizers.Encoding`.
+
+    Args:
+        model (:class:`~tokenizers.models.Model`):
+            The core algorithm that this :obj:`Tokenizer` should be using.
+
+    """
+    def __init__(self, model):
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    def add_special_tokens(self, tokens):
+        """
+        Add the given special tokens to the Tokenizer.
+
+        If these tokens are already part of the vocabulary, it just let the Tokenizer know about
+        them. If they don't exist, the Tokenizer creates them, giving them a new id.
+
+        These special tokens will never be processed by the model (ie won't be split into
+        multiple tokens), and they can be removed from the output when decoding.
+
+        Args:
+            tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`):
+                The list of special tokens we want to add to the vocabulary. Each token can either
+                be a string or an instance of :class:`~tokenizers.AddedToken` for more
+                customization.
+
+        Returns:
+            :obj:`int`: The number of tokens that were created in the vocabulary
+        """
+        pass
+
+    def add_tokens(self, tokens):
+        """
+        Add the given tokens to the vocabulary
+
+        The given tokens are added only if they don't already exist in the vocabulary.
+        Each token then gets a new attributed id.
+
+        Args:
+            tokens (A :obj:`List` of :class:`~tokenizers.AddedToken` or :obj:`str`):
+                The list of tokens we want to add to the vocabulary. Each token can be either a
+                string or an instance of :class:`~tokenizers.AddedToken` for more customization.
+
+        Returns:
+            :obj:`int`: The number of tokens that were created in the vocabulary
+        """
+        pass
+
+    def async_decode_batch(self, sequences, skip_special_tokens=True):
+        """
+        Decode a batch of ids back to their corresponding string
+
+        Args:
+            sequences (:obj:`List` of :obj:`List[int]`):
+                The batch of sequences we want to decode
+
+            skip_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether the special tokens should be removed from the decoded strings
+
+        Returns:
+            :obj:`List[str]`: A list of decoded strings
+        """
+        pass
+
+    def async_encode(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True):
+        """
+        Asynchronously encode the given input with character offsets.
+
+        This is an async version of encode that can be awaited in async Python code.
+
+        Example:
+            Here are some examples of the inputs that are accepted::
+
+                await async_encode("A single sequence")
+
+        Args:
+            sequence (:obj:`~tokenizers.InputSequence`):
+                The main input sequence we want to encode. This sequence can be either raw
+                text or pre-tokenized, according to the ``is_pretokenized`` argument:
+
+                - If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence`
+                - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence`
+
+            pair (:obj:`~tokenizers.InputSequence`, `optional`):
+                An optional input sequence. The expected format is the same that for ``sequence``.
+
+            is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
+                Whether the input is already pre-tokenized
+
+            add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to add the special tokens
+
+        Returns:
+            :class:`~tokenizers.Encoding`: The encoded result
+
+        """
+        pass
+
+    def async_encode_batch(self, input, is_pretokenized=False, add_special_tokens=True):
+        """
+        Asynchronously encode the given batch of inputs with character offsets.
+
+        This is an async version of encode_batch that can be awaited in async Python code.
+
+        Example:
+            Here are some examples of the inputs that are accepted::
+
+                await async_encode_batch([
+                    "A single sequence",
+                    ("A tuple with a sequence", "And its pair"),
+                    [ "A", "pre", "tokenized", "sequence" ],
+                    ([ "A", "pre", "tokenized", "sequence" ], "And its pair")
+                ])
+
+        Args:
+            input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
+                A list of single sequences or pair sequences to encode. Each sequence
+                can be either raw text or pre-tokenized, according to the ``is_pretokenized``
+                argument:
+
+                - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
+                - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
+
+            is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
+                Whether the input is already pre-tokenized
+
+            add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to add the special tokens
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
+
+        """
+        pass
+
+    def async_encode_batch_fast(self, input, is_pretokenized=False, add_special_tokens=True):
+        """
+        Asynchronously encode the given batch of inputs without tracking character offsets.
+
+        This is an async version of encode_batch_fast that can be awaited in async Python code.
+
+        Example:
+            Here are some examples of the inputs that are accepted::
+
+                await async_encode_batch_fast([
+                    "A single sequence",
+                    ("A tuple with a sequence", "And its pair"),
+                    [ "A", "pre", "tokenized", "sequence" ],
+                    ([ "A", "pre", "tokenized", "sequence" ], "And its pair")
+                ])
+
+        Args:
+            input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
+                A list of single sequences or pair sequences to encode. Each sequence
+                can be either raw text or pre-tokenized, according to the ``is_pretokenized``
+                argument:
+
+                - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
+                - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
+
+            is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
+                Whether the input is already pre-tokenized
+
+            add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to add the special tokens
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
+
+        """
+        pass
+
+    def decode(self, ids, skip_special_tokens=True):
+        """
+        Decode the given list of ids back to a string
+
+        This is used to decode anything coming back from a Language Model
+
+        Args:
+            ids (A :obj:`List/Tuple` of :obj:`int`):
+                The list of ids that we want to decode
+
+            skip_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether the special tokens should be removed from the decoded string
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+    def decode_batch(self, sequences, skip_special_tokens=True):
+        """
+        Decode a batch of ids back to their corresponding string
+
+        Args:
+            sequences (:obj:`List` of :obj:`List[int]`):
+                The batch of sequences we want to decode
+
+            skip_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether the special tokens should be removed from the decoded strings
+
+        Returns:
+            :obj:`List[str]`: A list of decoded strings
+        """
+        pass
+
+    @property
+    def decoder(self):
+        """
+        The `optional` :class:`~tokenizers.decoders.Decoder` in use by the Tokenizer
+        """
+        pass
+
+    @decoder.setter
+    def decoder(self, value):
+        """
+        The `optional` :class:`~tokenizers.decoders.Decoder` in use by the Tokenizer
+        """
+        pass
+
+    def enable_padding(
+        self, direction="right", pad_id=0, pad_type_id=0, pad_token="[PAD]", length=None, pad_to_multiple_of=None
+    ):
+        """
+        Enable the padding
+
+        Args:
+            direction (:obj:`str`, `optional`, defaults to :obj:`right`):
+                The direction in which to pad. Can be either ``right`` or ``left``
+
+            pad_to_multiple_of (:obj:`int`, `optional`):
+                If specified, the padding length should always snap to the next multiple of the
+                given value. For example if we were going to pad witha length of 250 but
+                ``pad_to_multiple_of=8`` then we will pad to 256.
+
+            pad_id (:obj:`int`, defaults to 0):
+                The id to be used when padding
+
+            pad_type_id (:obj:`int`, defaults to 0):
+                The type id to be used when padding
+
+            pad_token (:obj:`str`, defaults to :obj:`[PAD]`):
+                The pad token to be used when padding
+
+            length (:obj:`int`, `optional`):
+                If specified, the length at which to pad. If not specified we pad using the size of
+                the longest sequence in a batch.
+        """
+        pass
+
+    def enable_truncation(self, max_length, stride=0, strategy="longest_first", direction="right"):
+        """
+        Enable truncation
+
+        Args:
+            max_length (:obj:`int`):
+                The max length at which to truncate
+
+            stride (:obj:`int`, `optional`):
+                The length of the previous first sequence to be included in the overflowing
+                sequence
+
+            strategy (:obj:`str`, `optional`, defaults to :obj:`longest_first`):
+                The strategy used to truncation. Can be one of ``longest_first``, ``only_first`` or
+                ``only_second``.
+
+            direction (:obj:`str`, defaults to :obj:`right`):
+                Truncate direction
+        """
+        pass
+
+    def encode(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True):
+        """
+        Encode the given sequence and pair. This method can process raw text sequences
+        as well as already pre-tokenized sequences.
+
+        Example:
+            Here are some examples of the inputs that are accepted::
+
+                encode("A single sequence")`
+                encode("A sequence", "And its pair")`
+                encode([ "A", "pre", "tokenized", "sequence" ], is_pretokenized=True)`
+                encode(
+                    [ "A", "pre", "tokenized", "sequence" ], [ "And", "its", "pair" ],
+                    is_pretokenized=True
+                )
+
+        Args:
+            sequence (:obj:`~tokenizers.InputSequence`):
+                The main input sequence we want to encode. This sequence can be either raw
+                text or pre-tokenized, according to the ``is_pretokenized`` argument:
+
+                - If ``is_pretokenized=False``: :class:`~tokenizers.TextInputSequence`
+                - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedInputSequence`
+
+            pair (:obj:`~tokenizers.InputSequence`, `optional`):
+                An optional input sequence. The expected format is the same that for ``sequence``.
+
+            is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
+                Whether the input is already pre-tokenized
+
+            add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to add the special tokens
+
+        Returns:
+            :class:`~tokenizers.Encoding`: The encoded result
+
+        """
+        pass
+
+    def encode_batch(self, input, is_pretokenized=False, add_special_tokens=True):
+        """
+        Encode the given batch of inputs. This method accept both raw text sequences
+        as well as already pre-tokenized sequences. The reason we use `PySequence` is
+        because it allows type checking with zero-cost (according to PyO3) as we don't
+        have to convert to check.
+
+        Example:
+            Here are some examples of the inputs that are accepted::
+
+                encode_batch([
+                    "A single sequence",
+                    ("A tuple with a sequence", "And its pair"),
+                    [ "A", "pre", "tokenized", "sequence" ],
+                    ([ "A", "pre", "tokenized", "sequence" ], "And its pair")
+                ])
+
+        Args:
+            input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
+                A list of single sequences or pair sequences to encode. Each sequence
+                can be either raw text or pre-tokenized, according to the ``is_pretokenized``
+                argument:
+
+                - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
+                - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
+
+            is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
+                Whether the input is already pre-tokenized
+
+            add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to add the special tokens
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
+
+        """
+        pass
+
+    def encode_batch_fast(self, input, is_pretokenized=False, add_special_tokens=True):
+        """
+        Encode the given batch of inputs. This method is faster than `encode_batch`
+        because it doesn't keep track of offsets, they will be all zeros.
+
+        Example:
+            Here are some examples of the inputs that are accepted::
+
+                encode_batch_fast([
+                    "A single sequence",
+                    ("A tuple with a sequence", "And its pair"),
+                    [ "A", "pre", "tokenized", "sequence" ],
+                    ([ "A", "pre", "tokenized", "sequence" ], "And its pair")
+                ])
+
+        Args:
+            input (A :obj:`List`/:obj:`Tuple` of :obj:`~tokenizers.EncodeInput`):
+                A list of single sequences or pair sequences to encode. Each sequence
+                can be either raw text or pre-tokenized, according to the ``is_pretokenized``
+                argument:
+
+                - If ``is_pretokenized=False``: :class:`~tokenizers.TextEncodeInput`
+                - If ``is_pretokenized=True``: :class:`~tokenizers.PreTokenizedEncodeInput`
+
+            is_pretokenized (:obj:`bool`, defaults to :obj:`False`):
+                Whether the input is already pre-tokenized
+
+            add_special_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to add the special tokens
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch
+
+        """
+        pass
+
+    @property
+    def encode_special_tokens(self):
+        """
+        Modifies the tokenizer in order to use or not the special tokens
+        during encoding.
+
+        Args:
+            value (:obj:`bool`):
+                Whether to use the special tokens or not
+
+        """
+        pass
+
+    @encode_special_tokens.setter
+    def encode_special_tokens(self, value):
+        """
+        Modifies the tokenizer in order to use or not the special tokens
+        during encoding.
+
+        Args:
+            value (:obj:`bool`):
+                Whether to use the special tokens or not
+
+        """
+        pass
+
+    @staticmethod
+    def from_buffer(buffer):
+        """
+        Instantiate a new :class:`~tokenizers.Tokenizer` from the given buffer.
+
+        Args:
+            buffer (:obj:`bytes`):
+                A buffer containing a previously serialized :class:`~tokenizers.Tokenizer`
+
+        Returns:
+            :class:`~tokenizers.Tokenizer`: The new tokenizer
+        """
+        pass
+
+    @staticmethod
+    def from_file(path):
+        """
+        Instantiate a new :class:`~tokenizers.Tokenizer` from the file at the given path.
+
+        Args:
+            path (:obj:`str`):
+                A path to a local JSON file representing a previously serialized
+                :class:`~tokenizers.Tokenizer`
+
+        Returns:
+            :class:`~tokenizers.Tokenizer`: The new tokenizer
+        """
+        pass
+
+    @staticmethod
+    def from_pretrained(identifier, revision="main", token=None):
+        """
+        Instantiate a new :class:`~tokenizers.Tokenizer` from an existing file on the
+        Hugging Face Hub.
+
+        Args:
+            identifier (:obj:`str`):
+                The identifier of a Model on the Hugging Face Hub, that contains
+                a tokenizer.json file
+            revision (:obj:`str`, defaults to `main`):
+                A branch or commit id
+            token (:obj:`str`, `optional`, defaults to `None`):
+                An optional auth token used to access private repositories on the
+                Hugging Face Hub
+
+        Returns:
+            :class:`~tokenizers.Tokenizer`: The new tokenizer
+        """
+        pass
+
+    @staticmethod
+    def from_str(json):
+        """
+        Instantiate a new :class:`~tokenizers.Tokenizer` from the given JSON string.
+
+        Args:
+            json (:obj:`str`):
+                A valid JSON string representing a previously serialized
+                :class:`~tokenizers.Tokenizer`
+
+        Returns:
+            :class:`~tokenizers.Tokenizer`: The new tokenizer
+        """
+        pass
+
+    def get_added_tokens_decoder(self):
+        """
+        Get the underlying vocabulary
+
+        Returns:
+            :obj:`Dict[int, AddedToken]`: The vocabulary
+        """
+        pass
+
+    def get_vocab(self, with_added_tokens=True):
+        """
+        Get the underlying vocabulary
+
+        Args:
+            with_added_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to include the added tokens
+
+        Returns:
+            :obj:`Dict[str, int]`: The vocabulary
+        """
+        pass
+
+    def get_vocab_size(self, with_added_tokens=True):
+        """
+        Get the size of the underlying vocabulary
+
+        Args:
+            with_added_tokens (:obj:`bool`, defaults to :obj:`True`):
+                Whether to include the added tokens
+
+        Returns:
+            :obj:`int`: The size of the vocabulary
+        """
+        pass
+
+    def id_to_token(self, id):
+        """
+        Convert the given id to its corresponding token if it exists
+
+        Args:
+            id (:obj:`int`):
+                The id to convert
+
+        Returns:
+            :obj:`Optional[str]`: An optional token, :obj:`None` if out of vocabulary
+        """
+        pass
+
+    @property
+    def model(self):
+        """
+        The :class:`~tokenizers.models.Model` in use by the Tokenizer
+        """
+        pass
+
+    @model.setter
+    def model(self, value):
+        """
+        The :class:`~tokenizers.models.Model` in use by the Tokenizer
+        """
+        pass
+
+    def no_padding(self):
+        """
+        Disable padding
+        """
+        pass
+
+    def no_truncation(self):
+        """
+        Disable truncation
+        """
+        pass
+
+    @property
+    def normalizer(self):
+        """
+        The `optional` :class:`~tokenizers.normalizers.Normalizer` in use by the Tokenizer
+        """
+        pass
+
+    @normalizer.setter
+    def normalizer(self, value):
+        """
+        The `optional` :class:`~tokenizers.normalizers.Normalizer` in use by the Tokenizer
+        """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+        :param is_pair: Boolean indicating if the input would be a single sentence or a pair
+        :return:
+        """
+        pass
+
+    @property
+    def padding(self):
+        """
+        Get the current padding parameters
+
+        `Cannot be set, use` :meth:`~tokenizers.Tokenizer.enable_padding` `instead`
+
+        Returns:
+            (:obj:`dict`, `optional`):
+                A dict with the current padding parameters if padding is enabled
+        """
+        pass
+
+    @padding.setter
+    def padding(self, value):
+        """
+        Get the current padding parameters
+
+        `Cannot be set, use` :meth:`~tokenizers.Tokenizer.enable_padding` `instead`
+
+        Returns:
+            (:obj:`dict`, `optional`):
+                A dict with the current padding parameters if padding is enabled
+        """
+        pass
+
+    def post_process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Apply all the post-processing steps to the given encodings.
+
+        The various steps are:
+
+            1. Truncate according to the set truncation params (provided with
+               :meth:`~tokenizers.Tokenizer.enable_truncation`)
+            2. Apply the :class:`~tokenizers.processors.PostProcessor`
+            3. Pad according to the set padding params (provided with
+               :meth:`~tokenizers.Tokenizer.enable_padding`)
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The :class:`~tokenizers.Encoding` corresponding to the main sequence.
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                An optional :class:`~tokenizers.Encoding` corresponding to the pair sequence.
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Returns:
+            :class:`~tokenizers.Encoding`: The final post-processed encoding
+        """
+        pass
+
+    @property
+    def post_processor(self):
+        """
+        The `optional` :class:`~tokenizers.processors.PostProcessor` in use by the Tokenizer
+        """
+        pass
+
+    @post_processor.setter
+    def post_processor(self, value):
+        """
+        The `optional` :class:`~tokenizers.processors.PostProcessor` in use by the Tokenizer
+        """
+        pass
+
+    @property
+    def pre_tokenizer(self):
+        """
+        The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer
+        """
+        pass
+
+    @pre_tokenizer.setter
+    def pre_tokenizer(self, value):
+        """
+        The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer
+        """
+        pass
+
+    def save(self, path, pretty=True):
+        """
+        Save the :class:`~tokenizers.Tokenizer` to the file at the given path.
+
+        Args:
+            path (:obj:`str`):
+                A path to a file in which to save the serialized tokenizer.
+
+            pretty (:obj:`bool`, defaults to :obj:`True`):
+                Whether the JSON file should be pretty formatted.
+        """
+        pass
+
+    def to_str(self, pretty=False):
+        """
+        Gets a serialized string representing this :class:`~tokenizers.Tokenizer`.
+
+        Args:
+            pretty (:obj:`bool`, defaults to :obj:`False`):
+                Whether the JSON string should be pretty formatted.
+
+        Returns:
+            :obj:`str`: A string representing the serialized Tokenizer
+        """
+        pass
+
+    def token_to_id(self, token):
+        """
+        Convert the given token to its corresponding id if it exists
+
+        Args:
+            token (:obj:`str`):
+                The token to convert
+
+        Returns:
+            :obj:`Optional[int]`: An optional id, :obj:`None` if out of vocabulary
+        """
+        pass
+
+    def train(self, files, trainer=None):
+        """
+        Train the Tokenizer using the given files.
+
+        Reads the files line by line, while keeping all the whitespace, even new lines.
+        If you want to train from data store in-memory, you can check
+        :meth:`~tokenizers.Tokenizer.train_from_iterator`
+
+        Args:
+            files (:obj:`List[str]`):
+                A list of path to the files that we should use for training
+
+            trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`):
+                An optional trainer that should be used to train our Model
+        """
+        pass
+
+    def train_from_iterator(self, iterator, trainer=None, length=None):
+        """
+        Train the Tokenizer using the provided iterator.
+
+        You can provide anything that is a Python Iterator
+
+            * A list of sequences :obj:`List[str]`
+            * A generator that yields :obj:`str` or :obj:`List[str]`
+            * A Numpy array of strings
+            * ...
+
+        Args:
+            iterator (:obj:`Iterator`):
+                Any iterator over strings or list of strings
+
+            trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`):
+                An optional trainer that should be used to train our Model
+
+            length (:obj:`int`, `optional`):
+                The total number of sequences in the iterator. This is used to
+                provide meaningful progress tracking
+        """
+        pass
+
+    @property
+    def truncation(self):
+        """
+        Get the currently set truncation parameters
+
+        `Cannot set, use` :meth:`~tokenizers.Tokenizer.enable_truncation` `instead`
+
+        Returns:
+            (:obj:`dict`, `optional`):
+                A dict with the current truncation parameters if truncation is enabled
+        """
+        pass
+
+    @truncation.setter
+    def truncation(self, value):
+        """
+        Get the currently set truncation parameters
+
+        `Cannot set, use` :meth:`~tokenizers.Tokenizer.enable_truncation` `instead`
+
+        Returns:
+            (:obj:`dict`, `optional`):
+                A dict with the current truncation parameters if truncation is enabled
+        """
+        pass
+
+from enum import Enum
+from typing import List, Tuple, Union, Any
+
+Offsets = Tuple[int, int]
+TextInputSequence = str
+PreTokenizedInputSequence = Union[List[str], Tuple[str, ...]]
+TextEncodeInput = Union[
+    TextInputSequence,
+    Tuple[TextInputSequence, TextInputSequence],
+    List[TextInputSequence],
+]
+PreTokenizedEncodeInput = Union[
+    PreTokenizedInputSequence,
+    Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
+    List[PreTokenizedInputSequence],
+]
+InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
+EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
+
+class OffsetReferential(Enum):
+    ORIGINAL = "original"
+    NORMALIZED = "normalized"
+
+class OffsetType(Enum):
+    BYTE = "byte"
+    CHAR = "char"
+
+class SplitDelimiterBehavior(Enum):
+    REMOVED = "removed"
+    ISOLATED = "isolated"
+    MERGED_WITH_PREVIOUS = "merged_with_previous"
+    MERGED_WITH_NEXT = "merged_with_next"
+    CONTIGUOUS = "contiguous"
+
+from .implementations import (
+    BertWordPieceTokenizer,
+    ByteLevelBPETokenizer,
+    CharBPETokenizer,
+    SentencePieceBPETokenizer,
+    SentencePieceUnigramTokenizer,
+)
+
+def __getattr__(name: str) -> Any: ...
+
+BertWordPieceTokenizer: Any
+ByteLevelBPETokenizer: Any
+CharBPETokenizer: Any
+SentencePieceBPETokenizer: Any
+SentencePieceUnigramTokenizer: Any

source/tokenizers/decoders/__init__.py

@@ -0,0 +1,15 @@
+from .. import decoders
+
+
+Decoder = decoders.Decoder
+ByteLevel = decoders.ByteLevel
+Replace = decoders.Replace
+WordPiece = decoders.WordPiece
+ByteFallback = decoders.ByteFallback
+Fuse = decoders.Fuse
+Strip = decoders.Strip
+Metaspace = decoders.Metaspace
+BPEDecoder = decoders.BPEDecoder
+CTC = decoders.CTC
+Sequence = decoders.Sequence
+DecodeStream = decoders.DecodeStream

source/tokenizers/decoders/__init__.pyi

@@ -0,0 +1,569 @@
+# Generated content DO NOT EDIT
+class DecodeStream:
+    """
+    Class needed for streaming decode
+
+    """
+    def __init__(self, ids=None, skip_special_tokens=False):
+        pass
+
+    def __getstate__(self, /):
+        """
+        Helper for pickle.
+        """
+        pass
+
+    def step(self, tokenizer, id):
+        """
+        Streaming decode step
+
+        Args:
+            tokenizer (:class:`~tokenizers.Tokenizer`):
+               The tokenizer to use for decoding
+           id (:obj:`int` or `List[int]`):
+              The next token id or list of token ids to add to the stream
+
+
+        Returns:
+            :obj:`Optional[str]`: The next decoded string chunk, or None if not enough
+                tokens have been provided yet.
+        """
+        pass
+
+class Decoder:
+    """
+    Base class for all decoders
+
+    This class is not supposed to be instantiated directly. Instead, any implementation of
+    a Decoder will return an instance of this class when instantiated.
+    """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+class BPEDecoder(Decoder):
+    """
+    BPEDecoder Decoder
+
+    Args:
+        suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
+            The suffix that was used to characterize an end-of-word. This suffix will
+            be replaced by whitespaces during the decoding
+    """
+    def __init__(self, suffix="</w>"):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+    @property
+    def suffix(self):
+        """ """
+        pass
+
+    @suffix.setter
+    def suffix(self, value):
+        """ """
+        pass
+
+class ByteFallback(Decoder):
+    """
+    ByteFallback Decoder
+    ByteFallback is a simple trick which converts tokens looking like `<0x61>`
+    to pure bytes, and attempts to make them into a string. If the tokens
+    cannot be decoded you will get � instead for each inconvertible byte token
+
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+class ByteLevel(Decoder):
+    """
+    ByteLevel Decoder
+
+    This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.ByteLevel`
+    :class:`~tokenizers.pre_tokenizers.PreTokenizer`.
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+class CTC(Decoder):
+    """
+    CTC Decoder
+
+    Args:
+        pad_token (:obj:`str`, `optional`, defaults to :obj:`<pad>`):
+            The pad token used by CTC to delimit a new token.
+        word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`):
+            The word delimiter token. It will be replaced by a <space>
+        cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to cleanup some tokenization artifacts.
+            Mainly spaces before punctuation, and some abbreviated english forms.
+    """
+    def __init__(self, pad_token="<pad>", word_delimiter_token="|", cleanup=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def cleanup(self):
+        """ """
+        pass
+
+    @cleanup.setter
+    def cleanup(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+    @property
+    def pad_token(self):
+        """ """
+        pass
+
+    @pad_token.setter
+    def pad_token(self, value):
+        """ """
+        pass
+
+    @property
+    def word_delimiter_token(self):
+        """ """
+        pass
+
+    @word_delimiter_token.setter
+    def word_delimiter_token(self, value):
+        """ """
+        pass
+
+class Fuse(Decoder):
+    """
+    Fuse Decoder
+    Fuse simply fuses every token into a single string.
+    This is the last step of decoding, this decoder exists only if
+    there is need to add other decoders *after* the fusion
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+class Metaspace(Decoder):
+    """
+    Metaspace Decoder
+
+    Args:
+        replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
+            The replacement character. Must be exactly one character. By default we
+            use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
+
+        prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
+            Whether to add a space to the first word if there isn't already one. This
+            lets us treat `hello` exactly like `say hello`.
+            Choices: "always", "never", "first". First means the space is only added on the first
+            token (relevant when special tokens are used or other pre_tokenizer are used).
+    """
+    def __init__(self, replacement="▁", prepend_scheme="always", split=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+    @property
+    def prepend_scheme(self):
+        """ """
+        pass
+
+    @prepend_scheme.setter
+    def prepend_scheme(self, value):
+        """ """
+        pass
+
+    @property
+    def replacement(self):
+        """ """
+        pass
+
+    @replacement.setter
+    def replacement(self, value):
+        """ """
+        pass
+
+    @property
+    def split(self):
+        """ """
+        pass
+
+    @split.setter
+    def split(self, value):
+        """ """
+        pass
+
+class Replace(Decoder):
+    """
+    Replace Decoder
+
+    This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.Replace`
+    :class:`~tokenizers.pre_tokenizers.PreTokenizer`.
+    """
+    def __init__(self, pattern, content):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+class Sequence(Decoder):
+    """
+    Sequence Decoder
+
+    Args:
+        decoders (:obj:`List[Decoder]`)
+            The decoders that need to be chained
+    """
+    def __init__(self, decoders):
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+class Strip(Decoder):
+    """
+    Strip normalizer
+    Strips n left characters of each token, or n right characters of each token
+    """
+    def __init__(self, content=" ", left=0, right=0):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def content(self):
+        """ """
+        pass
+
+    @content.setter
+    def content(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+    @property
+    def start(self):
+        """ """
+        pass
+
+    @start.setter
+    def start(self, value):
+        """ """
+        pass
+
+    @property
+    def stop(self):
+        """ """
+        pass
+
+    @stop.setter
+    def stop(self, value):
+        """ """
+        pass
+
+class WordPiece(Decoder):
+    """
+    WordPiece Decoder
+
+    Args:
+        prefix (:obj:`str`, `optional`, defaults to :obj:`##`):
+            The prefix to use for subwords that are not a beginning-of-word
+
+        cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation,
+            and some abbreviated english forms.
+    """
+    def __init__(self, prefix="##", cleanup=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def cleanup(self):
+        """ """
+        pass
+
+    @cleanup.setter
+    def cleanup(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
+    def decode(self, tokens):
+        """
+        Decode the given list of tokens to a final string
+
+        Args:
+            tokens (:obj:`List[str]`):
+                The list of tokens to decode
+
+        Returns:
+            :obj:`str`: The decoded string
+        """
+        pass
+
+    @property
+    def prefix(self):
+        """ """
+        pass
+
+    @prefix.setter
+    def prefix(self, value):
+        """ """
+        pass

source/tokenizers/implementations/__init__.py

@@ -0,0 +1,6 @@
+from .base_tokenizer import BaseTokenizer
+from .bert_wordpiece import BertWordPieceTokenizer
+from .byte_level_bpe import ByteLevelBPETokenizer
+from .char_level_bpe import CharBPETokenizer
+from .sentencepiece_bpe import SentencePieceBPETokenizer
+from .sentencepiece_unigram import SentencePieceUnigramTokenizer

source/tokenizers/implementations/base_tokenizer.py

@@ -0,0 +1,459 @@
+from typing import Dict, List, Optional, Tuple, Union
+
+from tokenizers import AddedToken, EncodeInput, Encoding, InputSequence, Tokenizer
+from tokenizers.decoders import Decoder
+from tokenizers.models import Model
+from tokenizers.normalizers import Normalizer
+from tokenizers.pre_tokenizers import PreTokenizer
+from tokenizers.processors import PostProcessor
+
+
+Offsets = Tuple[int, int]
+
+
+class BaseTokenizer:
+    def __init__(self, tokenizer: Tokenizer, parameters=None):
+        self._tokenizer = tokenizer
+        self._parameters = parameters if parameters is not None else {}
+
+    def __repr__(self):
+        return "Tokenizer(vocabulary_size={}, {})".format(
+            self._tokenizer.get_vocab_size(),
+            ", ".join(k + "=" + str(v) for k, v in self._parameters.items()),
+        )
+
+    def num_special_tokens_to_add(self, is_pair: bool) -> int:
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+        :param is_pair: Boolean indicating if the input would be a single sentence or a pair
+        :return:
+        """
+        return self._tokenizer.num_special_tokens_to_add(is_pair)
+
+    def get_vocab(self, with_added_tokens: bool = True) -> Dict[str, int]:
+        """Returns the vocabulary
+
+        Args:
+            with_added_tokens: boolean:
+                Whether to include the added tokens in the vocabulary
+
+        Returns:
+            The vocabulary
+        """
+        return self._tokenizer.get_vocab(with_added_tokens=with_added_tokens)
+
+    def get_added_tokens_decoder(self) -> Dict[int, AddedToken]:
+        """Returns the added reverse vocabulary
+
+        Returns:
+            The added vocabulary mapping ints to AddedTokens
+        """
+        return self._tokenizer.get_added_tokens_decoder()
+
+    def get_vocab_size(self, with_added_tokens: bool = True) -> int:
+        """Return the size of vocabulary, with or without added tokens.
+
+        Args:
+            with_added_tokens: (`optional`) bool:
+                Whether to count in added special tokens or not
+
+        Returns:
+            Size of vocabulary
+        """
+        return self._tokenizer.get_vocab_size(with_added_tokens=with_added_tokens)
+
+    def enable_padding(
+        self,
+        direction: Optional[str] = "right",
+        pad_to_multiple_of: Optional[int] = None,
+        pad_id: Optional[int] = 0,
+        pad_type_id: Optional[int] = 0,
+        pad_token: Optional[str] = "[PAD]",
+        length: Optional[int] = None,
+    ):
+        """Change the padding strategy
+
+        Args:
+            direction: (`optional`) str:
+                Can be one of: `right` or `left`
+
+            pad_to_multiple_of: (`optional`) unsigned int:
+                If specified, the padding length should always snap to the next multiple of
+                the given value. For example if we were going to pad with a length of 250 but
+                `pad_to_multiple_of=8` then we will pad to 256.
+
+            pad_id: (`optional`) unsigned int:
+                The indice to be used when padding
+
+            pad_type_id: (`optional`) unsigned int:
+                The type indice to be used when padding
+
+            pad_token: (`optional`) str:
+                The pad token to be used when padding
+
+            length: (`optional`) unsigned int:
+                If specified, the length at which to pad. If not specified
+                we pad using the size of the longest sequence in a batch
+        """
+        return self._tokenizer.enable_padding(
+            direction=direction,
+            pad_to_multiple_of=pad_to_multiple_of,
+            pad_id=pad_id,
+            pad_type_id=pad_type_id,
+            pad_token=pad_token,
+            length=length,
+        )
+
+    def no_padding(self):
+        """Disable padding"""
+        return self._tokenizer.no_padding()
+
+    @property
+    def padding(self) -> Optional[dict]:
+        """Get the current padding parameters
+
+        Returns:
+            None if padding is disabled, a dict with the currently set parameters
+            if the padding is enabled.
+        """
+        return self._tokenizer.padding
+
+    def enable_truncation(self, max_length: int, stride: Optional[int] = 0, strategy: Optional[str] = "longest_first"):
+        """Change the truncation options
+
+        Args:
+            max_length: unsigned int:
+                The maximum length at which to truncate
+
+            stride: (`optional`) unsigned int:
+                The length of the previous first sequence to be included
+                in the overflowing sequence
+
+            strategy: (`optional`) str:
+                Can be one of `longest_first`, `only_first` or `only_second`
+        """
+        return self._tokenizer.enable_truncation(max_length, stride=stride, strategy=strategy)
+
+    def no_truncation(self):
+        """Disable truncation"""
+        return self._tokenizer.no_truncation()
+
+    @property
+    def truncation(self) -> Optional[dict]:
+        """Get the current truncation parameters
+
+        Returns:
+            None if truncation is disabled, a dict with the current truncation parameters if
+            truncation is enabled
+        """
+        return self._tokenizer.truncation
+
+    def add_tokens(self, tokens: List[Union[str, AddedToken]]) -> int:
+        """Add the given tokens to the vocabulary
+
+        Args:
+            tokens: List[Union[str, AddedToken]]:
+                A list of tokens to add to the vocabulary. Each token can either be
+                a string, or an instance of AddedToken
+
+        Returns:
+            The number of tokens that were added to the vocabulary
+        """
+        return self._tokenizer.add_tokens(tokens)
+
+    def add_special_tokens(self, special_tokens: List[Union[str, AddedToken]]) -> int:
+        """Add the given special tokens to the vocabulary, and treat them as special tokens.
+
+        The special tokens will never be processed by the model, and will be
+        removed while decoding.
+
+        Args:
+            tokens: List[Union[str, AddedToken]]:
+                A list of special tokens to add to the vocabulary. Each token can either be
+                a string, or an instance of AddedToken
+
+        Returns:
+            The number of tokens that were added to the vocabulary
+        """
+        return self._tokenizer.add_special_tokens(special_tokens)
+
+    def normalize(self, sequence: str) -> str:
+        """Normalize the given sequence
+
+        Args:
+            sequence: str:
+                The sequence to normalize
+
+        Returns:
+            The normalized string
+        """
+        return self._tokenizer.normalizer.normalize_str(sequence)
+
+    def encode(
+        self,
+        sequence: InputSequence,
+        pair: Optional[InputSequence] = None,
+        is_pretokenized: bool = False,
+        add_special_tokens: bool = True,
+    ) -> Encoding:
+        """Encode the given sequence and pair. This method can process raw text sequences as well
+        as already pre-tokenized sequences.
+
+        Args:
+            sequence: InputSequence:
+                The sequence we want to encode. This sequence can be either raw text or
+                pre-tokenized, according to the `is_pretokenized` argument:
+
+                - If `is_pretokenized=False`: `InputSequence` is expected to be `str`
+                - If `is_pretokenized=True`: `InputSequence` is expected to be
+                    `Union[List[str], Tuple[str]]`
+
+            is_pretokenized: bool:
+                Whether the input is already pre-tokenized.
+
+            add_special_tokens: bool:
+                Whether to add the special tokens while encoding.
+
+        Returns:
+            An Encoding
+        """
+        if sequence is None:
+            raise ValueError("encode: `sequence` can't be `None`")
+
+        return self._tokenizer.encode(sequence, pair, is_pretokenized, add_special_tokens)
+
+    def encode_batch(
+        self,
+        inputs: List[EncodeInput],
+        is_pretokenized: bool = False,
+        add_special_tokens: bool = True,
+    ) -> List[Encoding]:
+        """Encode the given inputs. This method accept both raw text sequences as well as already
+        pre-tokenized sequences.
+
+        Args:
+            inputs: List[EncodeInput]:
+                A list of single sequences or pair sequences to encode. Each `EncodeInput` is
+                expected to be of the following form:
+                    `Union[InputSequence, Tuple[InputSequence, InputSequence]]`
+
+                Each `InputSequence` can either be raw text or pre-tokenized,
+                according to the `is_pretokenized` argument:
+
+                - If `is_pretokenized=False`: `InputSequence` is expected to be `str`
+                - If `is_pretokenized=True`: `InputSequence` is expected to be
+                    `Union[List[str], Tuple[str]]`
+
+            is_pretokenized: bool:
+                Whether the input is already pre-tokenized.
+
+            add_special_tokens: bool:
+                Whether to add the special tokens while encoding.
+
+        Returns:
+            A list of Encoding
+        """
+
+        if inputs is None:
+            raise ValueError("encode_batch: `inputs` can't be `None`")
+
+        return self._tokenizer.encode_batch(inputs, is_pretokenized, add_special_tokens)
+
+    async def async_encode_batch(
+        self,
+        inputs: List[EncodeInput],
+        is_pretokenized: bool = False,
+        add_special_tokens: bool = True,
+    ) -> List[Encoding]:
+        """Asynchronously encode a batch (tracks character offsets).
+
+        Args:
+            inputs: A list of single or pair sequences to encode.
+            is_pretokenized: Whether inputs are already pre-tokenized.
+            add_special_tokens: Whether to add special tokens.
+
+        Returns:
+            A list of Encoding.
+        """
+        if inputs is None:
+            raise ValueError("async_encode_batch: `inputs` can't be `None`")
+        # Exposed by the Rust bindings via pyo3_async_runtimes::tokio::future_into_py
+        return await self._tokenizer.async_encode_batch(inputs, is_pretokenized, add_special_tokens)
+
+    async def async_encode_batch_fast(
+        self,
+        inputs: List[EncodeInput],
+        is_pretokenized: bool = False,
+        add_special_tokens: bool = True,
+    ) -> List[Encoding]:
+        """Asynchronously encode a batch (no character offsets, faster).
+
+        Args:
+            inputs: A list of single or pair sequences to encode.
+            is_pretokenized: Whether inputs are already pre-tokenized.
+            add_special_tokens: Whether to add special tokens.
+
+        Returns:
+            A list of Encoding.
+        """
+        if inputs is None:
+            raise ValueError("async_encode_batch_fast: `inputs` can't be `None`")
+        return await self._tokenizer.async_encode_batch_fast(inputs, is_pretokenized, add_special_tokens)
+
+    def decode(self, ids: List[int], skip_special_tokens: Optional[bool] = True) -> str:
+        """Decode the given list of ids to a string sequence
+
+        Args:
+            ids: List[unsigned int]:
+                A list of ids to be decoded
+
+            skip_special_tokens: (`optional`) boolean:
+                Whether to remove all the special tokens from the output string
+
+        Returns:
+            The decoded string
+        """
+        if ids is None:
+            raise ValueError("None input is not valid. Should be a list of integers.")
+
+        return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
+
+    def decode_batch(self, sequences: List[List[int]], skip_special_tokens: Optional[bool] = True) -> str:
+        """Decode the list of sequences to a list of string sequences
+
+        Args:
+            sequences: List[List[unsigned int]]:
+                A list of sequence of ids to be decoded
+
+            skip_special_tokens: (`optional`) boolean:
+                Whether to remove all the special tokens from the output strings
+
+        Returns:
+            A list of decoded strings
+        """
+        if sequences is None:
+            raise ValueError("None input is not valid. Should be list of list of integers.")
+
+        return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens)
+
+    def token_to_id(self, token: str) -> Optional[int]:
+        """Convert the given token to its corresponding id
+
+        Args:
+            token: str:
+                The token to convert
+
+        Returns:
+            The corresponding id if it exists, None otherwise
+        """
+        return self._tokenizer.token_to_id(token)
+
+    def id_to_token(self, id: int) -> Optional[str]:
+        """Convert the given token id to its corresponding string
+
+        Args:
+            token: id:
+                The token id to convert
+
+        Returns:
+            The corresponding string if it exists, None otherwise
+        """
+        return self._tokenizer.id_to_token(id)
+
+    def save_model(self, directory: str, prefix: Optional[str] = None):
+        """Save the current model to the given directory
+
+        Args:
+            directory: str:
+                A path to the destination directory
+
+            prefix: (Optional) str:
+                An optional prefix, used to prefix each file name
+        """
+        return self._tokenizer.model.save(directory, prefix=prefix)
+
+    def save(self, path: str, pretty: bool = True):
+        """Save the current Tokenizer at the given path
+
+        Args:
+            path: str:
+                A path to the destination Tokenizer file
+        """
+        return self._tokenizer.save(path, pretty)
+
+    def to_str(self, pretty: bool = False):
+        """Get a serialized JSON version of the Tokenizer as a str
+
+        Args:
+            pretty: bool:
+                Whether the JSON string should be prettified
+
+        Returns:
+            str
+        """
+        return self._tokenizer.to_str(pretty)
+
+    def post_process(
+        self, encoding: Encoding, pair: Optional[Encoding] = None, add_special_tokens: bool = True
+    ) -> Encoding:
+        """Apply all the post-processing steps to the given encodings.
+
+        The various steps are:
+            1. Truncate according to global params (provided to `enable_truncation`)
+            2. Apply the PostProcessor
+            3. Pad according to global params. (provided to `enable_padding`)
+
+        Args:
+            encoding: Encoding:
+                The main Encoding to post process
+
+            pair: Optional[Encoding]:
+                An optional pair Encoding
+
+            add_special_tokens: bool:
+                Whether to add special tokens
+
+        Returns:
+            The resulting Encoding
+        """
+        return self._tokenizer.post_process(encoding, pair, add_special_tokens)
+
+    @property
+    def model(self) -> Model:
+        return self._tokenizer.model
+
+    @model.setter
+    def model(self, model: Model):
+        self._tokenizer.model = model
+
+    @property
+    def normalizer(self) -> Normalizer:
+        return self._tokenizer.normalizer
+
+    @normalizer.setter
+    def normalizer(self, normalizer: Normalizer):
+        self._tokenizer.normalizer = normalizer
+
+    @property
+    def pre_tokenizer(self) -> PreTokenizer:
+        return self._tokenizer.pre_tokenizer
+
+    @pre_tokenizer.setter
+    def pre_tokenizer(self, pre_tokenizer: PreTokenizer):
+        self._tokenizer.pre_tokenizer = pre_tokenizer
+
+    @property
+    def post_processor(self) -> PostProcessor:
+        return self._tokenizer.post_processor
+
+    @post_processor.setter
+    def post_processor(self, post_processor: PostProcessor):
+        self._tokenizer.post_processor = post_processor
+
+    @property
+    def decoder(self) -> Decoder:
+        return self._tokenizer.decoder
+
+    @decoder.setter
+    def decoder(self, decoder: Decoder):
+        self._tokenizer.decoder = decoder

source/tokenizers/implementations/bert_wordpiece.py

@@ -0,0 +1,151 @@
+from typing import Dict, Iterator, List, Optional, Union
+
+from tokenizers import AddedToken, Tokenizer, decoders, trainers
+from tokenizers.models import WordPiece
+from tokenizers.normalizers import BertNormalizer
+from tokenizers.pre_tokenizers import BertPreTokenizer
+from tokenizers.processors import BertProcessing
+
+from .base_tokenizer import BaseTokenizer
+
+
+class BertWordPieceTokenizer(BaseTokenizer):
+    """Bert WordPiece Tokenizer"""
+
+    def __init__(
+        self,
+        vocab: Optional[Union[str, Dict[str, int]]] = None,
+        unk_token: Union[str, AddedToken] = "[UNK]",
+        sep_token: Union[str, AddedToken] = "[SEP]",
+        cls_token: Union[str, AddedToken] = "[CLS]",
+        pad_token: Union[str, AddedToken] = "[PAD]",
+        mask_token: Union[str, AddedToken] = "[MASK]",
+        clean_text: bool = True,
+        handle_chinese_chars: bool = True,
+        strip_accents: Optional[bool] = None,
+        lowercase: bool = True,
+        wordpieces_prefix: str = "##",
+    ):
+        if vocab is not None:
+            tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(unk_token)))
+        else:
+            tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token)))
+
+        # Let the tokenizer know about special tokens if they are part of the vocab
+        if tokenizer.token_to_id(str(unk_token)) is not None:
+            tokenizer.add_special_tokens([str(unk_token)])
+        if tokenizer.token_to_id(str(sep_token)) is not None:
+            tokenizer.add_special_tokens([str(sep_token)])
+        if tokenizer.token_to_id(str(cls_token)) is not None:
+            tokenizer.add_special_tokens([str(cls_token)])
+        if tokenizer.token_to_id(str(pad_token)) is not None:
+            tokenizer.add_special_tokens([str(pad_token)])
+        if tokenizer.token_to_id(str(mask_token)) is not None:
+            tokenizer.add_special_tokens([str(mask_token)])
+
+        tokenizer.normalizer = BertNormalizer(
+            clean_text=clean_text,
+            handle_chinese_chars=handle_chinese_chars,
+            strip_accents=strip_accents,
+            lowercase=lowercase,
+        )
+        tokenizer.pre_tokenizer = BertPreTokenizer()
+
+        if vocab is not None:
+            sep_token_id = tokenizer.token_to_id(str(sep_token))
+            if sep_token_id is None:
+                raise TypeError("sep_token not found in the vocabulary")
+            cls_token_id = tokenizer.token_to_id(str(cls_token))
+            if cls_token_id is None:
+                raise TypeError("cls_token not found in the vocabulary")
+
+            tokenizer.post_processor = BertProcessing((str(sep_token), sep_token_id), (str(cls_token), cls_token_id))
+        tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix)
+
+        parameters = {
+            "model": "BertWordPiece",
+            "unk_token": unk_token,
+            "sep_token": sep_token,
+            "cls_token": cls_token,
+            "pad_token": pad_token,
+            "mask_token": mask_token,
+            "clean_text": clean_text,
+            "handle_chinese_chars": handle_chinese_chars,
+            "strip_accents": strip_accents,
+            "lowercase": lowercase,
+            "wordpieces_prefix": wordpieces_prefix,
+        }
+
+        super().__init__(tokenizer, parameters)
+
+    @staticmethod
+    def from_file(vocab: str, **kwargs):
+        vocab = WordPiece.read_file(vocab)
+        return BertWordPieceTokenizer(vocab, **kwargs)
+
+    def train(
+        self,
+        files: Union[str, List[str]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        limit_alphabet: int = 1000,
+        initial_alphabet: List[str] = [],
+        special_tokens: List[Union[str, AddedToken]] = [
+            "[PAD]",
+            "[UNK]",
+            "[CLS]",
+            "[SEP]",
+            "[MASK]",
+        ],
+        show_progress: bool = True,
+        wordpieces_prefix: str = "##",
+    ):
+        """Train the model using the given files"""
+
+        trainer = trainers.WordPieceTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            limit_alphabet=limit_alphabet,
+            initial_alphabet=initial_alphabet,
+            special_tokens=special_tokens,
+            show_progress=show_progress,
+            continuing_subword_prefix=wordpieces_prefix,
+        )
+        if isinstance(files, str):
+            files = [files]
+        self._tokenizer.train(files, trainer=trainer)
+
+    def train_from_iterator(
+        self,
+        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        limit_alphabet: int = 1000,
+        initial_alphabet: List[str] = [],
+        special_tokens: List[Union[str, AddedToken]] = [
+            "[PAD]",
+            "[UNK]",
+            "[CLS]",
+            "[SEP]",
+            "[MASK]",
+        ],
+        show_progress: bool = True,
+        wordpieces_prefix: str = "##",
+        length: Optional[int] = None,
+    ):
+        """Train the model using the given iterator"""
+
+        trainer = trainers.WordPieceTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            limit_alphabet=limit_alphabet,
+            initial_alphabet=initial_alphabet,
+            special_tokens=special_tokens,
+            show_progress=show_progress,
+            continuing_subword_prefix=wordpieces_prefix,
+        )
+        self._tokenizer.train_from_iterator(
+            iterator,
+            trainer=trainer,
+            length=length,
+        )

source/tokenizers/implementations/byte_level_bpe.py

@@ -0,0 +1,122 @@
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, processors, trainers
+from tokenizers.models import BPE
+from tokenizers.normalizers import Lowercase, Sequence, unicode_normalizer_from_str
+
+from .base_tokenizer import BaseTokenizer
+
+
+class ByteLevelBPETokenizer(BaseTokenizer):
+    """ByteLevelBPETokenizer
+
+    Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
+    """
+
+    def __init__(
+        self,
+        vocab: Optional[Union[str, Dict[str, int]]] = None,
+        merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
+        add_prefix_space: bool = False,
+        lowercase: bool = False,
+        dropout: Optional[float] = None,
+        unicode_normalizer: Optional[str] = None,
+        continuing_subword_prefix: Optional[str] = None,
+        end_of_word_suffix: Optional[str] = None,
+        trim_offsets: bool = False,
+    ):
+        if vocab is not None and merges is not None:
+            tokenizer = Tokenizer(
+                BPE(
+                    vocab,
+                    merges,
+                    dropout=dropout,
+                    continuing_subword_prefix=continuing_subword_prefix or "",
+                    end_of_word_suffix=end_of_word_suffix or "",
+                )
+            )
+        else:
+            tokenizer = Tokenizer(BPE())
+
+        # Check for Unicode normalization first (before everything else)
+        normalizers = []
+
+        if unicode_normalizer:
+            normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
+
+        if lowercase:
+            normalizers += [Lowercase()]
+
+        # Create the normalizer structure
+        if len(normalizers) > 0:
+            if len(normalizers) > 1:
+                tokenizer.normalizer = Sequence(normalizers)
+            else:
+                tokenizer.normalizer = normalizers[0]
+
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=add_prefix_space)
+        tokenizer.decoder = decoders.ByteLevel()
+        tokenizer.post_processor = processors.ByteLevel(trim_offsets=trim_offsets)
+
+        parameters = {
+            "model": "ByteLevelBPE",
+            "add_prefix_space": add_prefix_space,
+            "lowercase": lowercase,
+            "dropout": dropout,
+            "unicode_normalizer": unicode_normalizer,
+            "continuing_subword_prefix": continuing_subword_prefix,
+            "end_of_word_suffix": end_of_word_suffix,
+            "trim_offsets": trim_offsets,
+        }
+
+        super().__init__(tokenizer, parameters)
+
+    @staticmethod
+    def from_file(vocab_filename: str, merges_filename: str, **kwargs):
+        vocab, merges = BPE.read_file(vocab_filename, merges_filename)
+        return ByteLevelBPETokenizer(vocab, merges, **kwargs)
+
+    def train(
+        self,
+        files: Union[str, List[str]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        show_progress: bool = True,
+        special_tokens: List[Union[str, AddedToken]] = [],
+    ):
+        """Train the model using the given files"""
+
+        trainer = trainers.BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            show_progress=show_progress,
+            special_tokens=special_tokens,
+            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+        )
+        if isinstance(files, str):
+            files = [files]
+        self._tokenizer.train(files, trainer=trainer)
+
+    def train_from_iterator(
+        self,
+        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        show_progress: bool = True,
+        special_tokens: List[Union[str, AddedToken]] = [],
+        length: Optional[int] = None,
+    ):
+        """Train the model using the given iterator"""
+
+        trainer = trainers.BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            show_progress=show_progress,
+            special_tokens=special_tokens,
+            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+        )
+        self._tokenizer.train_from_iterator(
+            iterator,
+            trainer=trainer,
+            length=length,
+        )

source/tokenizers/implementations/char_level_bpe.py

@@ -0,0 +1,150 @@
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+from .. import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers
+from ..models import BPE
+from ..normalizers import BertNormalizer, Lowercase, Sequence, unicode_normalizer_from_str
+from .base_tokenizer import BaseTokenizer
+
+
+class CharBPETokenizer(BaseTokenizer):
+    """Original BPE Tokenizer
+
+    Represents the BPE algorithm, as introduced by Rico Sennrich
+    (https://arxiv.org/abs/1508.07909)
+
+    The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
+    Sennrich subword-nmt implementation by the following options that you can deactivate:
+        - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
+            * removing any control characters and replacing all whitespaces by the classic one.
+            * handle chinese chars by putting spaces around them.
+            * strip all accents.
+        - spitting on punctuation in addition to whitespaces (deactivate it with
+          `split_on_whitespace_only=True`)
+    """
+
+    def __init__(
+        self,
+        vocab: Optional[Union[str, Dict[str, int]]] = None,
+        merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
+        unk_token: Union[str, AddedToken] = "<unk>",
+        suffix: str = "</w>",
+        dropout: Optional[float] = None,
+        lowercase: bool = False,
+        unicode_normalizer: Optional[str] = None,
+        bert_normalizer: bool = True,
+        split_on_whitespace_only: bool = False,
+    ):
+        if vocab is not None and merges is not None:
+            tokenizer = Tokenizer(
+                BPE(
+                    vocab,
+                    merges,
+                    dropout=dropout,
+                    unk_token=str(unk_token),
+                    end_of_word_suffix=suffix,
+                )
+            )
+        else:
+            tokenizer = Tokenizer(BPE(unk_token=str(unk_token), dropout=dropout, end_of_word_suffix=suffix))
+
+        if tokenizer.token_to_id(str(unk_token)) is not None:
+            tokenizer.add_special_tokens([str(unk_token)])
+
+        # Check for Unicode normalization first (before everything else)
+        normalizers = []
+
+        if unicode_normalizer:
+            normalizers += [unicode_normalizer_from_str(unicode_normalizer)]
+
+        if bert_normalizer:
+            normalizers += [BertNormalizer(lowercase=False)]
+
+        if lowercase:
+            normalizers += [Lowercase()]
+
+        # Create the normalizer structure
+        if len(normalizers) > 0:
+            if len(normalizers) > 1:
+                tokenizer.normalizer = Sequence(normalizers)
+            else:
+                tokenizer.normalizer = normalizers[0]
+
+        if split_on_whitespace_only:
+            tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+        else:
+            tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
+
+        tokenizer.decoder = decoders.BPEDecoder(suffix=suffix)
+
+        parameters = {
+            "model": "BPE",
+            "unk_token": unk_token,
+            "suffix": suffix,
+            "dropout": dropout,
+            "lowercase": lowercase,
+            "unicode_normalizer": unicode_normalizer,
+            "bert_normalizer": bert_normalizer,
+            "split_on_whitespace_only": split_on_whitespace_only,
+        }
+
+        super().__init__(tokenizer, parameters)
+
+    @staticmethod
+    def from_file(vocab_filename: str, merges_filename: str, **kwargs):
+        vocab, merges = BPE.read_file(vocab_filename, merges_filename)
+        return CharBPETokenizer(vocab, merges, **kwargs)
+
+    def train(
+        self,
+        files: Union[str, List[str]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
+        limit_alphabet: int = 1000,
+        initial_alphabet: List[str] = [],
+        suffix: Optional[str] = "</w>",
+        show_progress: bool = True,
+    ):
+        """Train the model using the given files"""
+
+        trainer = trainers.BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            special_tokens=special_tokens,
+            limit_alphabet=limit_alphabet,
+            initial_alphabet=initial_alphabet,
+            end_of_word_suffix=suffix,
+            show_progress=show_progress,
+        )
+        if isinstance(files, str):
+            files = [files]
+        self._tokenizer.train(files, trainer=trainer)
+
+    def train_from_iterator(
+        self,
+        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
+        limit_alphabet: int = 1000,
+        initial_alphabet: List[str] = [],
+        suffix: Optional[str] = "</w>",
+        show_progress: bool = True,
+        length: Optional[int] = None,
+    ):
+        """Train the model using the given iterator"""
+
+        trainer = trainers.BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            special_tokens=special_tokens,
+            limit_alphabet=limit_alphabet,
+            initial_alphabet=initial_alphabet,
+            end_of_word_suffix=suffix,
+            show_progress=show_progress,
+        )
+        self._tokenizer.train_from_iterator(
+            iterator,
+            trainer=trainer,
+            length=length,
+        )

source/tokenizers/implementations/sentencepiece_bpe.py

@@ -0,0 +1,103 @@
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers
+from tokenizers.models import BPE
+from tokenizers.normalizers import NFKC
+
+from .base_tokenizer import BaseTokenizer
+
+
+class SentencePieceBPETokenizer(BaseTokenizer):
+    """SentencePiece BPE Tokenizer
+
+    Represents the BPE algorithm, with the pretokenization used by SentencePiece
+    """
+
+    def __init__(
+        self,
+        vocab: Optional[Union[str, Dict[str, int]]] = None,
+        merges: Optional[Union[str, List[Tuple[str, str]]]] = None,
+        unk_token: Union[str, AddedToken] = "<unk>",
+        replacement: str = "▁",
+        add_prefix_space: bool = True,
+        dropout: Optional[float] = None,
+        fuse_unk: Optional[bool] = False,
+    ):
+        if vocab is not None and merges is not None:
+            tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk))
+        else:
+            tokenizer = Tokenizer(BPE(dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk))
+
+        if tokenizer.token_to_id(str(unk_token)) is not None:
+            tokenizer.add_special_tokens([str(unk_token)])
+
+        tokenizer.normalizer = NFKC()
+        prepend_scheme = "always" if add_prefix_space else "never"
+        tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
+        tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
+
+        parameters = {
+            "model": "SentencePieceBPE",
+            "unk_token": unk_token,
+            "replacement": replacement,
+            "add_prefix_space": add_prefix_space,
+            "dropout": dropout,
+        }
+
+        super().__init__(tokenizer, parameters)
+
+    @staticmethod
+    def from_file(vocab_filename: str, merges_filename: str, **kwargs):
+        vocab, merges = BPE.read_file(vocab_filename, merges_filename)
+        return SentencePieceBPETokenizer(vocab, merges, **kwargs)
+
+    def train(
+        self,
+        files: Union[str, List[str]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
+        limit_alphabet: int = 1000,
+        initial_alphabet: List[str] = [],
+        show_progress: bool = True,
+    ):
+        """Train the model using the given files"""
+
+        trainer = trainers.BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            special_tokens=special_tokens,
+            limit_alphabet=limit_alphabet,
+            initial_alphabet=initial_alphabet,
+            show_progress=show_progress,
+        )
+        if isinstance(files, str):
+            files = [files]
+        self._tokenizer.train(files, trainer=trainer)
+
+    def train_from_iterator(
+        self,
+        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
+        vocab_size: int = 30000,
+        min_frequency: int = 2,
+        special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
+        limit_alphabet: int = 1000,
+        initial_alphabet: List[str] = [],
+        show_progress: bool = True,
+        length: Optional[int] = None,
+    ):
+        """Train the model using the given iterator"""
+
+        trainer = trainers.BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=min_frequency,
+            special_tokens=special_tokens,
+            limit_alphabet=limit_alphabet,
+            initial_alphabet=initial_alphabet,
+            show_progress=show_progress,
+        )
+        self._tokenizer.train_from_iterator(
+            iterator,
+            trainer=trainer,
+            length=length,
+        )

source/tokenizers/implementations/sentencepiece_unigram.py

@@ -0,0 +1,196 @@
+import json
+import os
+from typing import Iterator, List, Optional, Union, Tuple
+
+from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
+from tokenizers.models import Unigram
+
+from .base_tokenizer import BaseTokenizer
+
+
+class SentencePieceUnigramTokenizer(BaseTokenizer):
+    """SentencePiece Unigram Tokenizer
+
+    Represents the Unigram algorithm, with the pretokenization used by SentencePiece
+    """
+
+    def __init__(
+        self,
+        vocab: Optional[List[Tuple[str, float]]] = None,
+        replacement: str = "▁",
+        add_prefix_space: bool = True,
+    ):
+        if vocab is not None:
+            # Let Unigram(..) fail if only one of them is None
+            tokenizer = Tokenizer(Unigram(vocab))
+        else:
+            tokenizer = Tokenizer(Unigram())
+
+        tokenizer.normalizer = normalizers.Sequence(
+            [normalizers.Nmt(), normalizers.NFKC(), normalizers.Replace(Regex(" {2,}"), " ")]
+        )
+        prepend_scheme = "always" if add_prefix_space else "never"
+        tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
+        tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
+
+        parameters = {
+            "model": "SentencePieceUnigram",
+            "replacement": replacement,
+            "add_prefix_space": add_prefix_space,
+        }
+
+        super().__init__(tokenizer, parameters)
+
+    def train(
+        self,
+        files: Union[str, List[str]],
+        vocab_size: int = 8000,
+        show_progress: bool = True,
+        special_tokens: Optional[List[Union[str, AddedToken]]] = None,
+        initial_alphabet: Optional[List[str]] = None,
+        unk_token: Optional[str] = None,
+    ):
+        """
+        Train the model using the given files
+
+        Args:
+            files (:obj:`List[str]`):
+                A list of path to the files that we should use for training
+            vocab_size (:obj:`int`):
+                The size of the final vocabulary, including all tokens and alphabet.
+            show_progress (:obj:`bool`):
+                Whether to show progress bars while training.
+            special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
+                A list of special tokens the model should know of.
+            initial_alphabet (:obj:`List[str]`, `optional`):
+                A list of characters to include in the initial alphabet, even
+                if not seen in the training dataset.
+                If the strings contain more than one character, only the first one
+                is kept.
+            unk_token (:obj:`str`, `optional`):
+                The unknown token to be used by the model.
+        """
+
+        if special_tokens is None:
+            special_tokens = []
+
+        if initial_alphabet is None:
+            initial_alphabet = []
+
+        trainer = trainers.UnigramTrainer(
+            vocab_size=vocab_size,
+            special_tokens=special_tokens,
+            show_progress=show_progress,
+            initial_alphabet=initial_alphabet,
+            unk_token=unk_token,
+        )
+
+        if isinstance(files, str):
+            files = [files]
+        self._tokenizer.train(files, trainer=trainer)
+
+    def train_from_iterator(
+        self,
+        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
+        vocab_size: int = 8000,
+        show_progress: bool = True,
+        special_tokens: Optional[List[Union[str, AddedToken]]] = None,
+        initial_alphabet: Optional[List[str]] = None,
+        unk_token: Optional[str] = None,
+        length: Optional[int] = None,
+    ):
+        """
+        Train the model using the given iterator
+
+        Args:
+            iterator (:obj:`Union[Iterator[str], Iterator[Iterator[str]]]`):
+                Any iterator over strings or list of strings
+            vocab_size (:obj:`int`):
+                The size of the final vocabulary, including all tokens and alphabet.
+            show_progress (:obj:`bool`):
+                Whether to show progress bars while training.
+            special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
+                A list of special tokens the model should know of.
+            initial_alphabet (:obj:`List[str]`, `optional`):
+                A list of characters to include in the initial alphabet, even
+                if not seen in the training dataset.
+                If the strings contain more than one character, only the first one
+                is kept.
+            unk_token (:obj:`str`, `optional`):
+                The unknown token to be used by the model.
+            length (:obj:`int`, `optional`):
+                The total number of sequences in the iterator. This is used to
+                provide meaningful progress tracking
+        """
+
+        if special_tokens is None:
+            special_tokens = []
+
+        if initial_alphabet is None:
+            initial_alphabet = []
+
+        trainer = trainers.UnigramTrainer(
+            vocab_size=vocab_size,
+            special_tokens=special_tokens,
+            show_progress=show_progress,
+            initial_alphabet=initial_alphabet,
+            unk_token=unk_token,
+        )
+
+        self._tokenizer.train_from_iterator(
+            iterator,
+            trainer=trainer,
+            length=length,
+        )
+
+    @staticmethod
+    def from_spm(filename: str):
+        try:
+            import sys
+
+            sys.path.append(".")
+
+            import sentencepiece_model_pb2 as model  # type: ignore[import]
+        except Exception:
+            raise Exception(
+                "You don't seem to have the required protobuf file, in order to use this function you need to run `pip install protobuf` and `wget https://raw.githubusercontent.com/google/sentencepiece/master/python/src/sentencepiece/sentencepiece_model_pb2.py` for us to be able to read the intrinsics of your spm_file. `pip install sentencepiece` is not required."
+            )
+
+        m = model.ModelProto()
+        m.ParseFromString(open(filename, "rb").read())
+
+        precompiled_charsmap = m.normalizer_spec.precompiled_charsmap
+        vocab = [(piece.piece, piece.score) for piece in m.pieces]
+        unk_id = m.trainer_spec.unk_id
+        model_type = m.trainer_spec.model_type
+        byte_fallback = m.trainer_spec.byte_fallback
+        if model_type != 1:
+            raise Exception(
+                "You're trying to run a `Unigram` model but you're file was trained with a different algorithm"
+            )
+
+        replacement = "▁"
+        add_prefix_space = True
+
+        tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback))
+
+        if precompiled_charsmap:
+            tokenizer.normalizer = normalizers.Sequence(
+                [
+                    normalizers.Precompiled(precompiled_charsmap),
+                    normalizers.Replace(Regex(" {2,}"), " "),
+                ]
+            )
+        else:
+            tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")])
+        prepend_scheme = "always" if add_prefix_space else "never"
+        tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
+        tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme)
+
+        parameters = {
+            "model": "SentencePieceUnigram",
+        }
+
+        obj = BaseTokenizer.__new__(SentencePieceUnigramTokenizer, tokenizer, parameters)  # type: ignore[arg-type]
+        BaseTokenizer.__init__(obj, tokenizer, parameters)
+        return obj

source/tokenizers/models/__init__.py

@@ -0,0 +1,8 @@
+# Generated content DO NOT EDIT
+from .. import models
+
+Model = models.Model
+BPE = models.BPE
+Unigram = models.Unigram
+WordLevel = models.WordLevel
+WordPiece = models.WordPiece

source/tokenizers/models/__init__.pyi

@@ -0,0 +1,744 @@
+# Generated content DO NOT EDIT
+class Model:
+    """
+    Base class for all models
+
+    The model represents the actual tokenization algorithm. This is the part that
+    will contain and manage the learned vocabulary.
+
+    This class cannot be constructed directly. Please use one of the concrete models.
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    def get_trainer(self):
+        """
+        Get the associated :class:`~tokenizers.trainers.Trainer`
+
+        Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
+        :class:`~tokenizers.models.Model`.
+
+        Returns:
+            :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
+        """
+        pass
+
+    def id_to_token(self, id):
+        """
+        Get the token associated to an ID
+
+        Args:
+            id (:obj:`int`):
+                An ID to convert to a token
+
+        Returns:
+            :obj:`str`: The token associated to the ID
+        """
+        pass
+
+    def save(self, folder, prefix):
+        """
+        Save the current model
+
+        Save the current model in the given folder, using the given prefix for the various
+        files that will get created.
+        Any file with the same name that already exists in this folder will be overwritten.
+
+        Args:
+            folder (:obj:`str`):
+                The path to the target folder in which to save the various files
+
+            prefix (:obj:`str`, `optional`):
+                An optional prefix, used to prefix each file name
+
+        Returns:
+            :obj:`List[str]`: The list of saved files
+        """
+        pass
+
+    def token_to_id(self, tokens):
+        """
+        Get the ID associated to a token
+
+        Args:
+            token (:obj:`str`):
+                A token to convert to an ID
+
+        Returns:
+            :obj:`int`: The ID associated to the token
+        """
+        pass
+
+    def tokenize(self, sequence):
+        """
+        Tokenize a sequence
+
+        Args:
+            sequence (:obj:`str`):
+                A sequence to tokenize
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
+        """
+        pass
+
+class BPE(Model):
+    """
+    An implementation of the BPE (Byte-Pair Encoding) algorithm
+
+    Args:
+        vocab (:obj:`Dict[str, int]`, `optional`):
+            A dictionary of string keys and their ids :obj:`{"am": 0,...}`
+
+        merges (:obj:`List[Tuple[str, str]]`, `optional`):
+            A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]`
+
+        cache_capacity (:obj:`int`, `optional`):
+            The number of words that the BPE cache can contain. The cache allows
+            to speed-up the process by keeping the result of the merge operations
+            for a number of words.
+
+        dropout (:obj:`float`, `optional`):
+            A float between 0 and 1 that represents the BPE dropout to use.
+
+        unk_token (:obj:`str`, `optional`):
+            The unknown token to be used by the model.
+
+        continuing_subword_prefix (:obj:`str`, `optional`):
+            The prefix to attach to subword units that don't represent a beginning of word.
+
+        end_of_word_suffix (:obj:`str`, `optional`):
+            The suffix to attach to subword units that represent an end of word.
+
+        fuse_unk (:obj:`bool`, `optional`):
+            Whether to fuse any subsequent unknown tokens into a single one
+
+        byte_fallback (:obj:`bool`, `optional`):
+            Whether to use spm byte-fallback trick (defaults to False)
+
+        ignore_merges (:obj:`bool`, `optional`):
+            Whether or not to match tokens with the vocab before using merges.
+    """
+    def __init__(
+        self,
+        vocab=None,
+        merges=None,
+        cache_capacity=None,
+        dropout=None,
+        unk_token=None,
+        continuing_subword_prefix=None,
+        end_of_word_suffix=None,
+        fuse_unk=None,
+        byte_fallback=False,
+        ignore_merges=False,
+    ):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def byte_fallback(self):
+        """ """
+        pass
+
+    @byte_fallback.setter
+    def byte_fallback(self, value):
+        """ """
+        pass
+
+    @property
+    def continuing_subword_prefix(self):
+        """ """
+        pass
+
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, value):
+        """ """
+        pass
+
+    @property
+    def dropout(self):
+        """ """
+        pass
+
+    @dropout.setter
+    def dropout(self, value):
+        """ """
+        pass
+
+    @property
+    def end_of_word_suffix(self):
+        """ """
+        pass
+
+    @end_of_word_suffix.setter
+    def end_of_word_suffix(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def from_file(vocab, merges, **kwargs):
+        """
+        Instantiate a BPE model from the given files.
+
+        This method is roughly equivalent to doing::
+
+           vocab, merges = BPE.read_file(vocab_filename, merges_filename)
+           bpe = BPE(vocab, merges)
+
+        If you don't need to keep the :obj:`vocab, merges` values lying around,
+        this method is more optimized than manually calling
+        :meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE`
+
+        Args:
+            vocab (:obj:`str`):
+                The path to a :obj:`vocab.json` file
+
+            merges (:obj:`str`):
+                The path to a :obj:`merges.txt` file
+
+        Returns:
+            :class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files
+        """
+        pass
+
+    @property
+    def fuse_unk(self):
+        """ """
+        pass
+
+    @fuse_unk.setter
+    def fuse_unk(self, value):
+        """ """
+        pass
+
+    def get_trainer(self):
+        """
+        Get the associated :class:`~tokenizers.trainers.Trainer`
+
+        Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
+        :class:`~tokenizers.models.Model`.
+
+        Returns:
+            :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
+        """
+        pass
+
+    def id_to_token(self, id):
+        """
+        Get the token associated to an ID
+
+        Args:
+            id (:obj:`int`):
+                An ID to convert to a token
+
+        Returns:
+            :obj:`str`: The token associated to the ID
+        """
+        pass
+
+    @property
+    def ignore_merges(self):
+        """ """
+        pass
+
+    @ignore_merges.setter
+    def ignore_merges(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def read_file(vocab, merges):
+        """
+        Read a :obj:`vocab.json` and a :obj:`merges.txt` files
+
+        This method provides a way to read and parse the content of these files,
+        returning the relevant data structures. If you want to instantiate some BPE models
+        from memory, this method gives you the expected input from the standard files.
+
+        Args:
+            vocab (:obj:`str`):
+                The path to a :obj:`vocab.json` file
+
+            merges (:obj:`str`):
+                The path to a :obj:`merges.txt` file
+
+        Returns:
+            A :obj:`Tuple` with the vocab and the merges:
+                The vocabulary and merges loaded into memory
+        """
+        pass
+
+    def save(self, folder, prefix):
+        """
+        Save the current model
+
+        Save the current model in the given folder, using the given prefix for the various
+        files that will get created.
+        Any file with the same name that already exists in this folder will be overwritten.
+
+        Args:
+            folder (:obj:`str`):
+                The path to the target folder in which to save the various files
+
+            prefix (:obj:`str`, `optional`):
+                An optional prefix, used to prefix each file name
+
+        Returns:
+            :obj:`List[str]`: The list of saved files
+        """
+        pass
+
+    def token_to_id(self, tokens):
+        """
+        Get the ID associated to a token
+
+        Args:
+            token (:obj:`str`):
+                A token to convert to an ID
+
+        Returns:
+            :obj:`int`: The ID associated to the token
+        """
+        pass
+
+    def tokenize(self, sequence):
+        """
+        Tokenize a sequence
+
+        Args:
+            sequence (:obj:`str`):
+                A sequence to tokenize
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
+        """
+        pass
+
+    @property
+    def unk_token(self):
+        """ """
+        pass
+
+    @unk_token.setter
+    def unk_token(self, value):
+        """ """
+        pass
+
+class Unigram(Model):
+    """
+    An implementation of the Unigram algorithm
+
+    Args:
+        vocab (:obj:`List[Tuple[str, float]]`, `optional`, `optional`):
+            A list of vocabulary items and their relative score [("am", -0.2442),...]
+    """
+    def __init__(self, vocab=None, unk_id=None, byte_fallback=None):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    def get_trainer(self):
+        """
+        Get the associated :class:`~tokenizers.trainers.Trainer`
+
+        Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
+        :class:`~tokenizers.models.Model`.
+
+        Returns:
+            :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
+        """
+        pass
+
+    def id_to_token(self, id):
+        """
+        Get the token associated to an ID
+
+        Args:
+            id (:obj:`int`):
+                An ID to convert to a token
+
+        Returns:
+            :obj:`str`: The token associated to the ID
+        """
+        pass
+
+    def save(self, folder, prefix):
+        """
+        Save the current model
+
+        Save the current model in the given folder, using the given prefix for the various
+        files that will get created.
+        Any file with the same name that already exists in this folder will be overwritten.
+
+        Args:
+            folder (:obj:`str`):
+                The path to the target folder in which to save the various files
+
+            prefix (:obj:`str`, `optional`):
+                An optional prefix, used to prefix each file name
+
+        Returns:
+            :obj:`List[str]`: The list of saved files
+        """
+        pass
+
+    def token_to_id(self, tokens):
+        """
+        Get the ID associated to a token
+
+        Args:
+            token (:obj:`str`):
+                A token to convert to an ID
+
+        Returns:
+            :obj:`int`: The ID associated to the token
+        """
+        pass
+
+    def tokenize(self, sequence):
+        """
+        Tokenize a sequence
+
+        Args:
+            sequence (:obj:`str`):
+                A sequence to tokenize
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
+        """
+        pass
+
+class WordLevel(Model):
+    """
+    An implementation of the WordLevel algorithm
+
+    Most simple tokenizer model based on mapping tokens to their corresponding id.
+
+    Args:
+        vocab (:obj:`str`, `optional`):
+            A dictionary of string keys and their ids :obj:`{"am": 0,...}`
+
+        unk_token (:obj:`str`, `optional`):
+            The unknown token to be used by the model.
+    """
+    def __init__(self, vocab=None, unk_token=None):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def from_file(vocab, unk_token=None):
+        """
+        Instantiate a WordLevel model from the given file
+
+        This method is roughly equivalent to doing::
+
+            vocab = WordLevel.read_file(vocab_filename)
+            wordlevel = WordLevel(vocab)
+
+        If you don't need to keep the :obj:`vocab` values lying around, this method is
+        more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to
+        initialize a :class:`~tokenizers.models.WordLevel`
+
+        Args:
+            vocab (:obj:`str`):
+                The path to a :obj:`vocab.json` file
+
+        Returns:
+            :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
+        """
+        pass
+
+    def get_trainer(self):
+        """
+        Get the associated :class:`~tokenizers.trainers.Trainer`
+
+        Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
+        :class:`~tokenizers.models.Model`.
+
+        Returns:
+            :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
+        """
+        pass
+
+    def id_to_token(self, id):
+        """
+        Get the token associated to an ID
+
+        Args:
+            id (:obj:`int`):
+                An ID to convert to a token
+
+        Returns:
+            :obj:`str`: The token associated to the ID
+        """
+        pass
+
+    @staticmethod
+    def read_file(vocab):
+        """
+        Read a :obj:`vocab.json`
+
+        This method provides a way to read and parse the content of a vocabulary file,
+        returning the relevant data structures. If you want to instantiate some WordLevel models
+        from memory, this method gives you the expected input from the standard files.
+
+        Args:
+            vocab (:obj:`str`):
+                The path to a :obj:`vocab.json` file
+
+        Returns:
+            :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
+        """
+        pass
+
+    def save(self, folder, prefix):
+        """
+        Save the current model
+
+        Save the current model in the given folder, using the given prefix for the various
+        files that will get created.
+        Any file with the same name that already exists in this folder will be overwritten.
+
+        Args:
+            folder (:obj:`str`):
+                The path to the target folder in which to save the various files
+
+            prefix (:obj:`str`, `optional`):
+                An optional prefix, used to prefix each file name
+
+        Returns:
+            :obj:`List[str]`: The list of saved files
+        """
+        pass
+
+    def token_to_id(self, tokens):
+        """
+        Get the ID associated to a token
+
+        Args:
+            token (:obj:`str`):
+                A token to convert to an ID
+
+        Returns:
+            :obj:`int`: The ID associated to the token
+        """
+        pass
+
+    def tokenize(self, sequence):
+        """
+        Tokenize a sequence
+
+        Args:
+            sequence (:obj:`str`):
+                A sequence to tokenize
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
+        """
+        pass
+
+    @property
+    def unk_token(self):
+        """ """
+        pass
+
+    @unk_token.setter
+    def unk_token(self, value):
+        """ """
+        pass
+
+class WordPiece(Model):
+    """
+    An implementation of the WordPiece algorithm
+
+    Args:
+        vocab (:obj:`Dict[str, int]`, `optional`):
+            A dictionary of string keys and their ids :obj:`{"am": 0,...}`
+
+        unk_token (:obj:`str`, `optional`):
+            The unknown token to be used by the model.
+
+        max_input_chars_per_word (:obj:`int`, `optional`):
+            The maximum number of characters to authorize in a single word.
+    """
+    def __init__(self, vocab=None, unk_token="[UNK]", max_input_chars_per_word=100, continuing_subword_prefix="##"):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def continuing_subword_prefix(self):
+        """ """
+        pass
+
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def from_file(vocab, **kwargs):
+        """
+        Instantiate a WordPiece model from the given file
+
+        This method is roughly equivalent to doing::
+
+            vocab = WordPiece.read_file(vocab_filename)
+            wordpiece = WordPiece(vocab)
+
+        If you don't need to keep the :obj:`vocab` values lying around, this method is
+        more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to
+        initialize a :class:`~tokenizers.models.WordPiece`
+
+        Args:
+            vocab (:obj:`str`):
+                The path to a :obj:`vocab.txt` file
+
+        Returns:
+            :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
+        """
+        pass
+
+    def get_trainer(self):
+        """
+        Get the associated :class:`~tokenizers.trainers.Trainer`
+
+        Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this
+        :class:`~tokenizers.models.Model`.
+
+        Returns:
+            :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model
+        """
+        pass
+
+    def id_to_token(self, id):
+        """
+        Get the token associated to an ID
+
+        Args:
+            id (:obj:`int`):
+                An ID to convert to a token
+
+        Returns:
+            :obj:`str`: The token associated to the ID
+        """
+        pass
+
+    @property
+    def max_input_chars_per_word(self):
+        """ """
+        pass
+
+    @max_input_chars_per_word.setter
+    def max_input_chars_per_word(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def read_file(vocab):
+        """
+        Read a :obj:`vocab.txt` file
+
+        This method provides a way to read and parse the content of a standard `vocab.txt`
+        file as used by the WordPiece Model, returning the relevant data structures. If you
+        want to instantiate some WordPiece models from memory, this method gives you the
+        expected input from the standard files.
+
+        Args:
+            vocab (:obj:`str`):
+                The path to a :obj:`vocab.txt` file
+
+        Returns:
+            :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict`
+        """
+        pass
+
+    def save(self, folder, prefix):
+        """
+        Save the current model
+
+        Save the current model in the given folder, using the given prefix for the various
+        files that will get created.
+        Any file with the same name that already exists in this folder will be overwritten.
+
+        Args:
+            folder (:obj:`str`):
+                The path to the target folder in which to save the various files
+
+            prefix (:obj:`str`, `optional`):
+                An optional prefix, used to prefix each file name
+
+        Returns:
+            :obj:`List[str]`: The list of saved files
+        """
+        pass
+
+    def token_to_id(self, tokens):
+        """
+        Get the ID associated to a token
+
+        Args:
+            token (:obj:`str`):
+                A token to convert to an ID
+
+        Returns:
+            :obj:`int`: The ID associated to the token
+        """
+        pass
+
+    def tokenize(self, sequence):
+        """
+        Tokenize a sequence
+
+        Args:
+            sequence (:obj:`str`):
+                A sequence to tokenize
+
+        Returns:
+            A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens
+        """
+        pass
+
+    @property
+    def unk_token(self):
+        """ """
+        pass
+
+    @unk_token.setter
+    def unk_token(self, value):
+        """ """
+        pass

source/tokenizers/normalizers/__init__.py

@@ -0,0 +1,29 @@
+from .. import normalizers
+
+
+Normalizer = normalizers.Normalizer
+BertNormalizer = normalizers.BertNormalizer
+NFD = normalizers.NFD
+NFKD = normalizers.NFKD
+NFC = normalizers.NFC
+NFKC = normalizers.NFKC
+Sequence = normalizers.Sequence
+Lowercase = normalizers.Lowercase
+Prepend = normalizers.Prepend
+Strip = normalizers.Strip
+StripAccents = normalizers.StripAccents
+Nmt = normalizers.Nmt
+Precompiled = normalizers.Precompiled
+Replace = normalizers.Replace
+ByteLevel = normalizers.ByteLevel
+
+NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD}
+
+
+def unicode_normalizer_from_str(normalizer: str) -> Normalizer:
+    if normalizer not in NORMALIZERS:
+        raise ValueError(
+            "{} is not a known unicode normalizer. Available are {}".format(normalizer, NORMALIZERS.keys())
+        )
+
+    return NORMALIZERS[normalizer]()

source/tokenizers/normalizers/__init__.pyi

@@ -0,0 +1,946 @@
+# Generated content DO NOT EDIT
+class Normalizer:
+    """
+    Base class for all normalizers
+
+    This class is not supposed to be instantiated directly. Instead, any implementation of a
+    Normalizer will return an instance of this class when instantiated.
+    """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class BertNormalizer(Normalizer):
+    """
+    BertNormalizer
+
+    Takes care of normalizing raw text before giving it to a Bert model.
+    This includes cleaning the text, handling accents, chinese chars and lowercasing
+
+    Args:
+        clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to clean the text, by removing any control characters
+            and replacing all whitespaces by the classic one.
+
+        handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to handle chinese chars by putting spaces around them.
+
+        strip_accents (:obj:`bool`, `optional`):
+            Whether to strip all accents. If this option is not specified (ie == None),
+            then it will be determined by the value for `lowercase` (as in the original Bert).
+
+        lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to lowercase.
+    """
+    def __init__(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def clean_text(self):
+        """ """
+        pass
+
+    @clean_text.setter
+    def clean_text(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    @property
+    def handle_chinese_chars(self):
+        """ """
+        pass
+
+    @handle_chinese_chars.setter
+    def handle_chinese_chars(self, value):
+        """ """
+        pass
+
+    @property
+    def lowercase(self):
+        """ """
+        pass
+
+    @lowercase.setter
+    def lowercase(self, value):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+    @property
+    def strip_accents(self):
+        """ """
+        pass
+
+    @strip_accents.setter
+    def strip_accents(self, value):
+        """ """
+        pass
+
+class ByteLevel(Normalizer):
+    """
+    Bytelevel Normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class Lowercase(Normalizer):
+    """
+    Lowercase Normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class NFC(Normalizer):
+    """
+    NFC Unicode Normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class NFD(Normalizer):
+    """
+    NFD Unicode Normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class NFKC(Normalizer):
+    """
+    NFKC Unicode Normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class NFKD(Normalizer):
+    """
+    NFKD Unicode Normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class Nmt(Normalizer):
+    """
+    Nmt normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class Precompiled(Normalizer):
+    """
+    Precompiled normalizer
+    Don't use manually it is used for compatibility for SentencePiece.
+    """
+    def __init__(self, precompiled_charsmap):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class Prepend(Normalizer):
+    """
+    Prepend normalizer
+    """
+    def __init__(self, prepend):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+    @property
+    def prepend(self):
+        """ """
+        pass
+
+    @prepend.setter
+    def prepend(self, value):
+        """ """
+        pass
+
+class Replace(Normalizer):
+    """
+    Replace normalizer
+    """
+    def __init__(self, pattern, content):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def content(self):
+        """ """
+        pass
+
+    @content.setter
+    def content(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+    @property
+    def pattern(self):
+        """ """
+        pass
+
+    @pattern.setter
+    def pattern(self, value):
+        """ """
+        pass
+
+class Sequence(Normalizer):
+    """
+    Allows concatenating multiple other Normalizer as a Sequence.
+    All the normalizers run in sequence in the given order
+
+    Args:
+        normalizers (:obj:`List[Normalizer]`):
+            A list of Normalizer to be run as a sequence
+    """
+    def __init__(self, normalizers):
+        pass
+
+    def __getitem__(self, key):
+        """
+        Return self[key].
+        """
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setitem__(self, key, value):
+        """
+        Set self[key] to value.
+        """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+class Strip(Normalizer):
+    """
+    Strip normalizer
+    """
+    def __init__(self, left=True, right=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    @property
+    def left(self):
+        """ """
+        pass
+
+    @left.setter
+    def left(self, value):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+    @property
+    def right(self):
+        """ """
+        pass
+
+    @right.setter
+    def right(self, value):
+        """ """
+        pass
+
+class StripAccents(Normalizer):
+    """
+    StripAccents normalizer
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    def normalize(self, normalized):
+        """
+        Normalize a :class:`~tokenizers.NormalizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.NormalizedString` to
+        keep track of the alignment information. If you just want to see the result
+        of the normalization on a raw string, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize_str`
+
+        Args:
+            normalized (:class:`~tokenizers.NormalizedString`):
+                The normalized string on which to apply this
+                :class:`~tokenizers.normalizers.Normalizer`
+        """
+        pass
+
+    def normalize_str(self, sequence):
+        """
+        Normalize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment
+        information. If you need to get/convert offsets, you can use
+        :meth:`~tokenizers.normalizers.Normalizer.normalize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to normalize
+
+        Returns:
+            :obj:`str`: A string after normalization
+        """
+        pass
+
+from typing import Dict
+
+NORMALIZERS: Dict[str, Normalizer]
+
+def unicode_normalizer_from_str(normalizer: str) -> Normalizer: ...

source/tokenizers/pre_tokenizers/__init__.py

@@ -0,0 +1,16 @@
+# Generated content DO NOT EDIT
+from .. import pre_tokenizers
+
+PreTokenizer = pre_tokenizers.PreTokenizer
+BertPreTokenizer = pre_tokenizers.BertPreTokenizer
+ByteLevel = pre_tokenizers.ByteLevel
+CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit
+Digits = pre_tokenizers.Digits
+FixedLength = pre_tokenizers.FixedLength
+Metaspace = pre_tokenizers.Metaspace
+Punctuation = pre_tokenizers.Punctuation
+Sequence = pre_tokenizers.Sequence
+Split = pre_tokenizers.Split
+UnicodeScripts = pre_tokenizers.UnicodeScripts
+Whitespace = pre_tokenizers.Whitespace
+WhitespaceSplit = pre_tokenizers.WhitespaceSplit

source/tokenizers/pre_tokenizers/__init__.pyi

@@ -0,0 +1,1015 @@
+# Generated content DO NOT EDIT
+class PreTokenizer:
+    """
+    Base class for all pre-tokenizers
+
+    This class is not supposed to be instantiated directly. Instead, any implementation of a
+    PreTokenizer will return an instance of this class when instantiated.
+    """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class BertPreTokenizer(PreTokenizer):
+    """
+    BertPreTokenizer
+
+    This pre-tokenizer splits tokens on spaces, and also on punctuation.
+    Each occurrence of a punctuation character will be treated separately.
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class ByteLevel(PreTokenizer):
+    """
+    ByteLevel PreTokenizer
+
+    This pre-tokenizer takes care of replacing all bytes of the given string
+    with a corresponding representation, as well as splitting into words.
+
+    Args:
+        add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to add a space to the first word if there isn't already one. This
+            lets us treat `hello` exactly like `say hello`.
+        use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Set this to :obj:`False` to prevent this `pre_tokenizer` from using
+            the GPT2 specific regexp for spliting on whitespace.
+    """
+    def __init__(self, add_prefix_space=True, trim_offsets=True, use_regex=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def add_prefix_space(self):
+        """ """
+        pass
+
+    @add_prefix_space.setter
+    def add_prefix_space(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def alphabet():
+        """
+        Returns the alphabet used by this PreTokenizer.
+
+        Since the ByteLevel works as its name suggests, at the byte level, it
+        encodes each byte value to a unique visible character. This means that there is a
+        total of 256 different characters composing this alphabet.
+
+        Returns:
+            :obj:`List[str]`: A list of characters that compose the alphabet
+        """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+    @property
+    def trim_offsets(self):
+        """ """
+        pass
+
+    @trim_offsets.setter
+    def trim_offsets(self, value):
+        """ """
+        pass
+
+    @property
+    def use_regex(self):
+        """ """
+        pass
+
+    @use_regex.setter
+    def use_regex(self, value):
+        """ """
+        pass
+
+class CharDelimiterSplit(PreTokenizer):
+    """
+    This pre-tokenizer simply splits on the provided char. Works like `.split(delimiter)`
+
+    Args:
+        delimiter: str:
+            The delimiter char that will be used to split input
+    """
+    def __init__(self, delimiter):
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    @property
+    def delimiter(self):
+        """ """
+        pass
+
+    @delimiter.setter
+    def delimiter(self, value):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class Digits(PreTokenizer):
+    """
+    This pre-tokenizer simply splits using the digits in separate tokens
+
+    Args:
+        individual_digits (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If set to True, digits will each be separated as follows::
+
+                "Call 123 please" -> "Call ", "1", "2", "3", " please"
+
+            If set to False, digits will grouped as follows::
+
+                "Call 123 please" -> "Call ", "123", " please"
+    """
+    def __init__(self, individual_digits=False):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    @property
+    def individual_digits(self):
+        """ """
+        pass
+
+    @individual_digits.setter
+    def individual_digits(self, value):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class FixedLength(PreTokenizer):
+    """
+    This pre-tokenizer splits the text into fixed length chunks as used
+    [here](https://www.biorxiv.org/content/10.1101/2023.01.11.523679v1.full)
+
+    Args:
+        length (:obj:`int`, `optional`, defaults to :obj:`5`):
+            The length of the chunks to split the text into.
+
+            Strings are split on the character level rather than the byte level to avoid
+            splitting unicode characters consisting of multiple bytes.
+    """
+    def __init__(self, length=5):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    @property
+    def length(self):
+        """ """
+        pass
+
+    @length.setter
+    def length(self, value):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class Metaspace(PreTokenizer):
+    """
+    Metaspace pre-tokenizer
+
+    This pre-tokenizer replaces any whitespace by the provided replacement character.
+    It then tries to split on these spaces.
+
+    Args:
+        replacement (:obj:`str`, `optional`, defaults to :obj:`▁`):
+            The replacement character. Must be exactly one character. By default we
+            use the `▁` (U+2581) meta symbol (Same as in SentencePiece).
+
+        prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`):
+            Whether to add a space to the first word if there isn't already one. This
+            lets us treat `hello` exactly like `say hello`.
+            Choices: "always", "never", "first". First means the space is only added on the first
+            token (relevant when special tokens are used or other pre_tokenizer are used).
+
+    """
+    def __init__(self, replacement="_", prepend_scheme="always", split=True):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+    @property
+    def prepend_scheme(self):
+        """ """
+        pass
+
+    @prepend_scheme.setter
+    def prepend_scheme(self, value):
+        """ """
+        pass
+
+    @property
+    def replacement(self):
+        """ """
+        pass
+
+    @replacement.setter
+    def replacement(self, value):
+        """ """
+        pass
+
+    @property
+    def split(self):
+        """ """
+        pass
+
+    @split.setter
+    def split(self, value):
+        """ """
+        pass
+
+class Punctuation(PreTokenizer):
+    """
+    This pre-tokenizer simply splits on punctuation as individual characters.
+
+    Args:
+        behavior (:class:`~tokenizers.SplitDelimiterBehavior`):
+            The behavior to use when splitting.
+            Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next",
+            "contiguous"
+    """
+    def __init__(self, behavior="isolated"):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def behavior(self):
+        """ """
+        pass
+
+    @behavior.setter
+    def behavior(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class Sequence(PreTokenizer):
+    """
+    This pre-tokenizer composes other pre_tokenizers and applies them in sequence
+    """
+    def __init__(self, pretokenizers):
+        pass
+
+    def __getitem__(self, key):
+        """
+        Return self[key].
+        """
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setitem__(self, key, value):
+        """
+        Set self[key] to value.
+        """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class Split(PreTokenizer):
+    """
+    Split PreTokenizer
+
+    This versatile pre-tokenizer splits using the provided pattern and
+    according to the provided behavior. The pattern can be inverted by
+    making use of the invert flag.
+
+    Args:
+        pattern (:obj:`str` or :class:`~tokenizers.Regex`):
+            A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`.
+            If you want to use a regex pattern, it has to be wrapped around a `tokenizers.Regex`,
+            otherwise we consider is as a string pattern. For example `pattern="|"`
+            means you want to split on `|` (imagine a csv file for example), while
+            `pattern=tokenizers.Regex("1|2")` means you split on either '1' or '2'.
+        behavior (:class:`~tokenizers.SplitDelimiterBehavior`):
+            The behavior to use when splitting.
+            Choices: "removed", "isolated", "merged_with_previous", "merged_with_next",
+            "contiguous"
+
+        invert (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to invert the pattern.
+    """
+    def __init__(self, pattern, behavior, invert=False):
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def behavior(self):
+        """ """
+        pass
+
+    @behavior.setter
+    def behavior(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    @property
+    def invert(self):
+        """ """
+        pass
+
+    @invert.setter
+    def invert(self, value):
+        """ """
+        pass
+
+    @property
+    def pattern(self):
+        """ """
+        pass
+
+    @pattern.setter
+    def pattern(self, value):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class UnicodeScripts(PreTokenizer):
+    """
+    This pre-tokenizer splits on characters that belong to different language family
+    It roughly follows https://github.com/google/sentencepiece/blob/master/data/Scripts.txt
+    Actually Hiragana and Katakana are fused with Han, and 0x30FC is Han too.
+    This mimicks SentencePiece Unigram implementation.
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class Whitespace(PreTokenizer):
+    """
+    This pre-tokenizer splits on word boundaries according to the `\w+|[^\w\s]+`
+    regex pattern. It splits on word characters or characters that aren't words or
+    whitespaces (punctuation such as hyphens, apostrophes, commas, etc.).
+
+    Example:
+        Use the `Whitespace` function as shown below::
+
+            ```python
+            from tokenizers.pre_tokenizers import Whitespace
+
+            pre_tokenizer = Whitespace()
+            text = "Hello, world! Let's try the Whitespace pre-tokenizer."
+            pre_tokenizer.pre_tokenize_str(text)
+            [('Hello', (0, 5)),
+             (',', (5, 6)),
+             ('world', (7, 12)),
+             ('!', (12, 13)),
+             ('Let', (14, 17)),
+             ("'", (17, 18)),
+             ('s', (18, 19)),
+             ('try', (20, 23)),
+             ('the', (24, 27)),
+             ('Whitespace', (28, 38)),
+             ('pre', (39, 42)),
+             ('-', (42, 43)),
+             ('tokenizer', (43, 52)),
+             ('.', (52, 53))]
+            ```
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass
+
+class WhitespaceSplit(PreTokenizer):
+    """
+    This pre-tokenizer simply splits on the whitespace. Works like `.split()`
+    """
+    def __init__(self):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(pretok):
+        """ """
+        pass
+
+    def pre_tokenize(self, pretok):
+        """
+        Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place
+
+        This method allows to modify a :class:`~tokenizers.PreTokenizedString` to
+        keep track of the pre-tokenization, and leverage the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of
+        the pre-tokenization of a raw string, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str`
+
+        Args:
+            pretok (:class:`~tokenizers.PreTokenizedString):
+                The pre-tokenized string on which to apply this
+                :class:`~tokenizers.pre_tokenizers.PreTokenizer`
+        """
+        pass
+
+    def pre_tokenize_str(self, sequence):
+        """
+        Pre tokenize the given string
+
+        This method provides a way to visualize the effect of a
+        :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the
+        alignment, nor does it provide all the capabilities of the
+        :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use
+        :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize`
+
+        Args:
+            sequence (:obj:`str`):
+                A string to pre-tokeize
+
+        Returns:
+            :obj:`List[Tuple[str, Offsets]]`:
+                A list of tuple with the pre-tokenized parts and their offsets
+        """
+        pass

source/tokenizers/processors/__init__.py

@@ -0,0 +1,9 @@
+# Generated content DO NOT EDIT
+from .. import processors
+
+PostProcessor = processors.PostProcessor
+BertProcessing = processors.BertProcessing
+ByteLevel = processors.ByteLevel
+RobertaProcessing = processors.RobertaProcessing
+Sequence = processors.Sequence
+TemplateProcessing = processors.TemplateProcessing

source/tokenizers/processors/__init__.pyi

@@ -0,0 +1,519 @@
+# Generated content DO NOT EDIT
+class PostProcessor:
+    """
+    Base class for all post-processors
+
+    This class is not supposed to be instantiated directly. Instead, any implementation of
+    a PostProcessor will return an instance of this class when instantiated.
+    """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+
+        Args:
+            is_pair (:obj:`bool`):
+                Whether the input would be a pair of sequences
+
+        Returns:
+            :obj:`int`: The number of tokens to add
+        """
+        pass
+
+    def process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Post-process the given encodings, generating the final one
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding for the first sequence
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                The encoding for the pair sequence
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Return:
+            :class:`~tokenizers.Encoding`: The final encoding
+        """
+        pass
+
+class BertProcessing(PostProcessor):
+    """
+    This post-processor takes care of adding the special tokens needed by
+    a Bert model:
+
+        - a SEP token
+        - a CLS token
+
+    Args:
+        sep (:obj:`Tuple[str, int]`):
+            A tuple with the string representation of the SEP token, and its id
+
+        cls (:obj:`Tuple[str, int]`):
+            A tuple with the string representation of the CLS token, and its id
+    """
+    def __init__(self, sep, cls):
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def cls(self):
+        """ """
+        pass
+
+    @cls.setter
+    def cls(self, value):
+        """ """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+
+        Args:
+            is_pair (:obj:`bool`):
+                Whether the input would be a pair of sequences
+
+        Returns:
+            :obj:`int`: The number of tokens to add
+        """
+        pass
+
+    def process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Post-process the given encodings, generating the final one
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding for the first sequence
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                The encoding for the pair sequence
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Return:
+            :class:`~tokenizers.Encoding`: The final encoding
+        """
+        pass
+
+    @property
+    def sep(self):
+        """ """
+        pass
+
+    @sep.setter
+    def sep(self, value):
+        """ """
+        pass
+
+class ByteLevel(PostProcessor):
+    """
+    This post-processor takes care of trimming the offsets.
+
+    By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
+    want the offsets to include these whitespaces, then this PostProcessor must be used.
+
+    Args:
+        trim_offsets (:obj:`bool`):
+            Whether to trim the whitespaces from the produced offsets.
+
+        add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments
+            the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets`
+            is set to :obj:`True`.
+    """
+    def __init__(self, add_prefix_space=None, trim_offsets=None, use_regex=None):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def add_prefix_space(self):
+        """ """
+        pass
+
+    @add_prefix_space.setter
+    def add_prefix_space(self, value):
+        """ """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+
+        Args:
+            is_pair (:obj:`bool`):
+                Whether the input would be a pair of sequences
+
+        Returns:
+            :obj:`int`: The number of tokens to add
+        """
+        pass
+
+    def process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Post-process the given encodings, generating the final one
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding for the first sequence
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                The encoding for the pair sequence
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Return:
+            :class:`~tokenizers.Encoding`: The final encoding
+        """
+        pass
+
+    @property
+    def trim_offsets(self):
+        """ """
+        pass
+
+    @trim_offsets.setter
+    def trim_offsets(self, value):
+        """ """
+        pass
+
+    @property
+    def use_regex(self):
+        """ """
+        pass
+
+    @use_regex.setter
+    def use_regex(self, value):
+        """ """
+        pass
+
+class RobertaProcessing(PostProcessor):
+    """
+    This post-processor takes care of adding the special tokens needed by
+    a Roberta model:
+
+        - a SEP token
+        - a CLS token
+
+    It also takes care of trimming the offsets.
+    By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
+    want the offsets to include these whitespaces, then this PostProcessor should be initialized
+    with :obj:`trim_offsets=True`
+
+    Args:
+        sep (:obj:`Tuple[str, int]`):
+            A tuple with the string representation of the SEP token, and its id
+
+        cls (:obj:`Tuple[str, int]`):
+            A tuple with the string representation of the CLS token, and its id
+
+        trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether to trim the whitespaces from the produced offsets.
+
+        add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether the add_prefix_space option was enabled during pre-tokenization. This
+            is relevant because it defines the way the offsets are trimmed out.
+    """
+    def __init__(self, sep, cls, trim_offsets=True, add_prefix_space=True):
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def add_prefix_space(self):
+        """ """
+        pass
+
+    @add_prefix_space.setter
+    def add_prefix_space(self, value):
+        """ """
+        pass
+
+    @property
+    def cls(self):
+        """ """
+        pass
+
+    @cls.setter
+    def cls(self, value):
+        """ """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+
+        Args:
+            is_pair (:obj:`bool`):
+                Whether the input would be a pair of sequences
+
+        Returns:
+            :obj:`int`: The number of tokens to add
+        """
+        pass
+
+    def process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Post-process the given encodings, generating the final one
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding for the first sequence
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                The encoding for the pair sequence
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Return:
+            :class:`~tokenizers.Encoding`: The final encoding
+        """
+        pass
+
+    @property
+    def sep(self):
+        """ """
+        pass
+
+    @sep.setter
+    def sep(self, value):
+        """ """
+        pass
+
+    @property
+    def trim_offsets(self):
+        """ """
+        pass
+
+    @trim_offsets.setter
+    def trim_offsets(self, value):
+        """ """
+        pass
+
+class Sequence(PostProcessor):
+    """
+    Sequence Processor
+
+    Args:
+        processors (:obj:`List[PostProcessor]`)
+            The processors that need to be chained
+    """
+    def __init__(self, processors):
+        pass
+
+    def __getitem__(self, key):
+        """
+        Return self[key].
+        """
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setitem__(self, key, value):
+        """
+        Set self[key] to value.
+        """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+
+        Args:
+            is_pair (:obj:`bool`):
+                Whether the input would be a pair of sequences
+
+        Returns:
+            :obj:`int`: The number of tokens to add
+        """
+        pass
+
+    def process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Post-process the given encodings, generating the final one
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding for the first sequence
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                The encoding for the pair sequence
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Return:
+            :class:`~tokenizers.Encoding`: The final encoding
+        """
+        pass
+
+class TemplateProcessing(PostProcessor):
+    """
+    Provides a way to specify templates in order to add the special tokens to each
+    input sequence as relevant.
+
+    Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to
+    delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first
+    sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair
+    sequences. The final result looks like this:
+
+        - Single sequence: :obj:`[CLS] Hello there [SEP]`
+        - Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]`
+
+    With the type ids as following::
+
+        [CLS]   ...   [SEP]   ...   [SEP]
+          0      0      0      1      1
+
+    You can achieve such behavior using a TemplateProcessing::
+
+        TemplateProcessing(
+            single="[CLS] $0 [SEP]",
+            pair="[CLS] $A [SEP] $B:1 [SEP]:1",
+            special_tokens=[("[CLS]", 1), ("[SEP]", 0)],
+        )
+
+    In this example, each input sequence is identified using a ``$`` construct. This identifier
+    lets us specify each input sequence, and the type_id to use. When nothing is specified,
+    it uses the default values. Here are the different ways to specify it:
+
+        - Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B``
+        - Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ...
+        - Specifying both: ``$A:0``, ``$B:1``, ...
+
+    The same construct is used for special tokens: ``<identifier>(:<type_id>)?``.
+
+    **Warning**: You must ensure that you are giving the correct tokens/ids as these
+    will be added to the Encoding without any further check. If the given ids correspond
+    to something totally different in a `Tokenizer` using this `PostProcessor`, it
+    might lead to unexpected results.
+
+    Args:
+        single (:obj:`Template`):
+            The template used for single sequences
+
+        pair (:obj:`Template`):
+            The template used when both sequences are specified
+
+        special_tokens (:obj:`Tokens`):
+            The list of special tokens used in each sequences
+
+    Types:
+
+        Template (:obj:`str` or :obj:`List`):
+            - If a :obj:`str` is provided, the whitespace is used as delimiter between tokens
+            - If a :obj:`List[str]` is provided, a list of tokens
+
+        Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`):
+            - A :obj:`Tuple` with both a token and its associated ID, in any order
+            - A :obj:`dict` with the following keys:
+                - "id": :obj:`str` => The special token id, as specified in the Template
+                - "ids": :obj:`List[int]` => The associated IDs
+                - "tokens": :obj:`List[str]` => The associated tokens
+
+             The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have
+             the same length.
+    """
+    def __init__(self, single=None, pair=None, special_tokens=None):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    def num_special_tokens_to_add(self, is_pair):
+        """
+        Return the number of special tokens that would be added for single/pair sentences.
+
+        Args:
+            is_pair (:obj:`bool`):
+                Whether the input would be a pair of sequences
+
+        Returns:
+            :obj:`int`: The number of tokens to add
+        """
+        pass
+
+    def process(self, encoding, pair=None, add_special_tokens=True):
+        """
+        Post-process the given encodings, generating the final one
+
+        Args:
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding for the first sequence
+
+            pair (:class:`~tokenizers.Encoding`, `optional`):
+                The encoding for the pair sequence
+
+            add_special_tokens (:obj:`bool`):
+                Whether to add the special tokens
+
+        Return:
+            :class:`~tokenizers.Encoding`: The final encoding
+        """
+        pass
+
+    @property
+    def single(self):
+        """ """
+        pass
+
+    @single.setter
+    def single(self, value):
+        """ """
+        pass

source/tokenizers/tokenizers.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c116fcf1e80d461ce0a35c332974f25949e8359416f50b3d53371810d2ce1ccc
+size 10074176

source/tokenizers/tokenizers.pyi

@@ -0,0 +1,17 @@
+# Generated content DO NOT EDIT
+from . import (
+    AddedToken as AddedToken,
+    Encoding as Encoding,
+    NormalizedString as NormalizedString,
+    PreTokenizedString as PreTokenizedString,
+    Regex as Regex,
+    Token as Token,
+    Tokenizer as Tokenizer,
+    __version__ as __version__,
+    decoders as decoders,
+    models as models,
+    normalizers as normalizers,
+    pre_tokenizers as pre_tokenizers,
+    processors as processors,
+    trainers as trainers,
+)

source/tokenizers/tools/__init__.py

@@ -0,0 +1 @@
+from .visualizer import Annotation, EncodingVisualizer

source/tokenizers/tools/visualizer-styles.css

@@ -0,0 +1,170 @@
+.tokenized-text {
+    width:100%;
+    padding:2rem;
+    max-height: 400px;
+    overflow-y: auto;
+    box-sizing:border-box;
+    line-height:4rem; /* Lots of space between lines */
+    font-family: "Roboto Light", "Ubuntu Light", "Ubuntu", monospace;
+    box-shadow: 2px 2px 2px rgba(0,0,0,0.2);
+    background-color: rgba(0,0,0,0.01);
+    letter-spacing:2px; /* Give some extra separation between chars */
+}
+.non-token{
+    /* White space and other things the tokenizer ignores*/
+    white-space: pre;
+    letter-spacing:4px;
+    border-top:1px solid #A0A0A0; /* A gentle border on top and bottom makes tabs more ovious*/
+    border-bottom:1px solid #A0A0A0;
+    line-height: 1rem;
+    height: calc(100% - 2px);
+}
+
+.token {
+    white-space: pre;
+    position:relative;
+    color:black;
+    letter-spacing:2px;
+}
+
+.annotation{
+    white-space:nowrap; /* Important - ensures that annotations appears even if the annotated text wraps a line */
+    border-radius:4px;
+    position:relative;
+    width:fit-content;
+}
+.annotation:before {
+    /*The before holds the text and the after holds the background*/
+    z-index:1000; /* Make sure this is above the background */
+    content:attr(data-label); /* The annotations label is on a data attribute */
+    color:white;
+    position:absolute;
+    font-size:1rem;
+    text-align:center;
+    font-weight:bold;
+
+    top:1.75rem;
+    line-height:0;
+    left:0;
+    width:100%;
+    padding:0.5rem 0;
+    /* These make it so an annotation doesn't stretch beyond the annotated text if the label is longer*/
+    overflow: hidden;
+    white-space: nowrap;
+    text-overflow:ellipsis;
+}
+
+.annotation:after {
+    content:attr(data-label); /* The content defines the width of the annotation*/
+    position:absolute;
+    font-size:0.75rem;
+    text-align:center;
+    font-weight:bold;
+    text-overflow:ellipsis;
+    top:1.75rem;
+    line-height:0;
+    overflow: hidden;
+    white-space: nowrap;
+
+    left:0;
+    width:100%; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/
+
+    padding:0.5rem 0;
+    /* Nast hack below:
+    We set the annotations color in code because we don't know the colors at css time.
+    But you can't pass a color as a data attribute to get it into the pseudo element (this thing)
+    So to get around that, annotations have the color set on them with a style attribute and then we
+    can get the color with currentColor.
+    Annotations wrap tokens and tokens set the color back to black
+     */
+    background-color: currentColor;
+}
+.annotation:hover::after, .annotation:hover::before{
+    /* When the user hovers over an annotation expand the label to display in full
+     */
+    min-width: fit-content;
+}
+
+.annotation:hover{
+    /* Emphasize the annotation start end with a border on hover*/
+    border-color: currentColor;
+    border: 2px solid;
+}
+.special-token:not(:empty){
+    /*
+    A none empty special token is like UNK (as opposed to CLS which has no representation in the text )
+     */
+    position:relative;
+}
+.special-token:empty::before{
+    /* Special tokens that don't have text are displayed as pseudo elements so we dont select them with the mouse*/
+    content:attr(data-stok);
+    background:#202020;
+    font-size:0.75rem;
+    color:white;
+    margin: 0 0.25rem;
+    padding: 0.25rem;
+    border-radius:4px
+}
+
+.special-token:not(:empty):before {
+    /* Special tokens that have text (UNK) are displayed above the actual text*/
+    content:attr(data-stok);
+    position:absolute;
+    bottom:1.75rem;
+    min-width:100%;
+    width:100%;
+    height:1rem;
+    line-height:1rem;
+    font-size:1rem;
+    text-align:center;
+    color:white;
+    font-weight:bold;
+    background:#202020;
+    border-radius:10%;
+}
+/*
+We want to alternate the color of tokens, but we can't use nth child because tokens might be broken up by annotations
+instead we apply even and odd class at generation time and color them that way
+ */
+.even-token{
+    background:#DCDCDC	;
+    border: 1px solid #DCDCDC;
+}
+.odd-token{
+    background:#A0A0A0;
+    border: 1px solid #A0A0A0;
+}
+.even-token.multi-token,.odd-token.multi-token{
+    background:  repeating-linear-gradient(
+    45deg,
+    transparent,
+    transparent 1px,
+    #ccc 1px,
+    #ccc 1px
+    ),
+    /* on "bottom" */
+    linear-gradient(
+    to bottom,
+    #FFB6C1,
+    #999
+    );
+}
+
+.multi-token:hover::after {
+    content:"This char has more than 1 token"; /* The content defines the width of the annotation*/
+    color:white;
+    background-color: black;
+    position:absolute;
+    font-size:0.75rem;
+    text-align:center;
+    font-weight:bold;
+    text-overflow:ellipsis;
+    top:1.75rem;
+    line-height:0;
+    overflow: hidden;
+    white-space: nowrap;
+    left:0;
+    width:fit-content; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/
+    padding:0.5rem 0;
+}

source/tokenizers/tools/visualizer.py

@@ -0,0 +1,407 @@
+import itertools
+import os
+import re
+from string import Template
+from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple
+
+from tokenizers import Encoding, Tokenizer
+
+
+dirname = os.path.dirname(__file__)
+css_filename = os.path.join(dirname, "visualizer-styles.css")
+with open(css_filename) as f:
+    css = f.read()
+
+
+class Annotation:
+    start: int
+    end: int
+    label: str
+
+    def __init__(self, start: int, end: int, label: str):
+        self.start = start
+        self.end = end
+        self.label = label
+
+
+AnnotationList = List[Annotation]
+PartialIntList = List[Optional[int]]
+
+
+class CharStateKey(NamedTuple):
+    token_ix: Optional[int]
+    anno_ix: Optional[int]
+
+
+class CharState:
+    char_ix: Optional[int]
+
+    def __init__(self, char_ix):
+        self.char_ix = char_ix
+
+        self.anno_ix: Optional[int] = None
+        self.tokens: List[int] = []
+
+    @property
+    def token_ix(self):
+        return self.tokens[0] if len(self.tokens) > 0 else None
+
+    @property
+    def is_multitoken(self):
+        """
+        BPE tokenizers can output more than one token for a char
+        """
+        return len(self.tokens) > 1
+
+    def partition_key(self) -> CharStateKey:
+        return CharStateKey(
+            token_ix=self.token_ix,
+            anno_ix=self.anno_ix,
+        )
+
+
+class Aligned:
+    pass
+
+
+class EncodingVisualizer:
+    """
+    Build an EncodingVisualizer
+
+    Args:
+
+         tokenizer (:class:`~tokenizers.Tokenizer`):
+            A tokenizer instance
+
+         default_to_notebook (:obj:`bool`):
+            Whether to render html output in a notebook by default
+
+         annotation_converter (:obj:`Callable`, `optional`):
+            An optional (lambda) function that takes an annotation in any format and returns
+            an Annotation object
+    """
+
+    unk_token_regex = re.compile("(.{1}\b)?(unk|oov)(\b.{1})?", flags=re.IGNORECASE)
+
+    def __init__(
+        self,
+        tokenizer: Tokenizer,
+        default_to_notebook: bool = True,
+        annotation_converter: Optional[Callable[[Any], Annotation]] = None,
+    ):
+        if default_to_notebook:
+            try:
+                from IPython.core.display import HTML, display  # type: ignore[attr-defined]
+            except ImportError:
+                raise Exception(
+                    """We couldn't import IPython utils for html display.
+                        Are you running in a notebook?
+                        You can also pass `default_to_notebook=False` to get back raw HTML
+                    """
+                )
+
+        self.tokenizer = tokenizer
+        self.default_to_notebook = default_to_notebook
+        self.annotation_coverter = annotation_converter
+        pass
+
+    def __call__(
+        self,
+        text: str,
+        annotations: Optional[List[Any]] = None,
+        default_to_notebook: Optional[bool] = None,
+    ) -> Optional[str]:
+        """
+        Build a visualization of the given text
+
+        Args:
+            text (:obj:`str`):
+                The text to tokenize
+
+            annotations (:obj:`List[Annotation]`, `optional`):
+                An optional list of annotations of the text. The can either be an annotation class
+                or anything else if you instantiated the visualizer with a converter function
+
+            default_to_notebook (:obj:`bool`, `optional`, defaults to `False`):
+                If True, will render the html in a notebook. Otherwise returns an html string.
+
+        Returns:
+            The HTML string if default_to_notebook is False, otherwise (default) returns None and
+            renders the HTML in the notebook
+
+        """
+        final_default_to_notebook = self.default_to_notebook
+        if default_to_notebook is not None:
+            final_default_to_notebook = default_to_notebook
+        if final_default_to_notebook:
+            try:
+                from IPython.core.display import HTML, display  # type: ignore[attr-defined]
+            except ImportError:
+                raise Exception(
+                    """We couldn't import IPython utils for html display.
+                    Are you running in a notebook?"""
+                )
+        if annotations is None:
+            annotations = []
+        if self.annotation_coverter is not None:
+            annotations = list(map(self.annotation_coverter, annotations))
+        encoding = self.tokenizer.encode(text)
+        html = EncodingVisualizer.__make_html(text, encoding, annotations)
+        if final_default_to_notebook:
+            display(HTML(html))
+        else:
+            return html
+
+    @staticmethod
+    def calculate_label_colors(annotations: AnnotationList) -> Dict[str, str]:
+        """
+        Generates a color palette for all the labels in a given set of annotations
+
+        Args:
+          annotations (:obj:`Annotation`):
+            A list of annotations
+
+        Returns:
+            :obj:`dict`: A dictionary mapping labels to colors in HSL format
+        """
+        if len(annotations) == 0:
+            return {}
+        labels = set(map(lambda x: x.label, annotations))
+        num_labels = len(labels)
+        h_step = int(255 / num_labels)
+        if h_step < 20:
+            h_step = 20
+        s = 32
+        l = 64  # noqa: E741
+        h = 10
+        colors = {}
+
+        for label in sorted(labels):  # sort so we always get the same colors for a given set of labels
+            colors[label] = f"hsl({h},{s}%,{l}%)"
+            h += h_step
+        return colors
+
+    @staticmethod
+    def consecutive_chars_to_html(
+        consecutive_chars_list: List[CharState],
+        text: str,
+        encoding: Encoding,
+    ):
+        """
+        Converts a list of "consecutive chars" into a single HTML element.
+        Chars are consecutive if they fall under the same word, token and annotation.
+        The CharState class is a named tuple with a "partition_key" method that makes it easy to
+        compare if two chars are consecutive.
+
+        Args:
+            consecutive_chars_list (:obj:`List[CharState]`):
+                A list of CharStates that have been grouped together
+
+            text (:obj:`str`):
+                The original text being processed
+
+            encoding (:class:`~tokenizers.Encoding`):
+                The encoding returned from the tokenizer
+
+        Returns:
+            :obj:`str`: The HTML span for a set of consecutive chars
+        """
+        first = consecutive_chars_list[0]
+        if first.char_ix is None:
+            # its a special token
+            stoken = encoding.tokens[first.token_ix]
+            # special tokens are represented as empty spans. We use the data attribute and css
+            # magic to display it
+            return f'<span class="special-token" data-stoken={stoken}></span>'
+        # We're not in a special token so this group has a start and end.
+        last = consecutive_chars_list[-1]
+        assert first.char_ix is not None
+        assert last.char_ix is not None
+        start = first.char_ix
+        end = last.char_ix + 1
+        span_text = text[start:end]
+        css_classes = []  # What css classes will we apply on the resulting span
+        data_items = {}  # What data attributes will we apply on the result span
+        if first.token_ix is not None:
+            # We can either be in a token or not (e.g. in white space)
+            css_classes.append("token")
+            if first.is_multitoken:
+                css_classes.append("multi-token")
+            if first.token_ix % 2:
+                # We use this to color alternating tokens.
+                # A token might be split by an annotation that ends in the middle of it, so this
+                # lets us visually indicate a consecutive token despite its possible splitting in
+                # the html markup
+                css_classes.append("odd-token")
+            else:
+                # Like above, but a different color so we can see the tokens alternate
+                css_classes.append("even-token")
+            if EncodingVisualizer.unk_token_regex.search(encoding.tokens[first.token_ix]) is not None:
+                # This is a special token that is in the text. probably UNK
+                css_classes.append("special-token")
+                # TODO is this the right name for the data attribute ?
+                data_items["stok"] = encoding.tokens[first.token_ix]
+        else:
+            # In this case we are looking at a group/single char that is not tokenized.
+            # e.g. white space
+            css_classes.append("non-token")
+        css = f'''class="{" ".join(css_classes)}"'''
+        data = ""
+        for key, val in data_items.items():
+            data += f' data-{key}="{val}"'
+        return f"<span {css} {data} >{span_text}</span>"
+
+    @staticmethod
+    def __make_html(text: str, encoding: Encoding, annotations: AnnotationList) -> str:
+        char_states = EncodingVisualizer.__make_char_states(text, encoding, annotations)
+        current_consecutive_chars = [char_states[0]]
+        prev_anno_ix = char_states[0].anno_ix
+        spans = []
+        label_colors_dict = EncodingVisualizer.calculate_label_colors(annotations)
+        cur_anno_ix = char_states[0].anno_ix
+        if cur_anno_ix is not None:
+            # If we started in an  annotation make a span for it
+            anno = annotations[cur_anno_ix]
+            label = anno.label
+            color = label_colors_dict[label]
+            spans.append(f'<span class="annotation" style="color:{color}" data-label="{label}">')
+
+        for cs in char_states[1:]:
+            cur_anno_ix = cs.anno_ix
+            if cur_anno_ix != prev_anno_ix:
+                # If we've transitioned in or out of an annotation
+                spans.append(
+                    # Create a span from the current consecutive characters
+                    EncodingVisualizer.consecutive_chars_to_html(
+                        current_consecutive_chars,
+                        text=text,
+                        encoding=encoding,
+                    )
+                )
+                current_consecutive_chars = [cs]
+
+                if prev_anno_ix is not None:
+                    # if we transitioned out of an annotation close it's span
+                    spans.append("</span>")
+                if cur_anno_ix is not None:
+                    # If we entered a new annotation make a span for it
+                    anno = annotations[cur_anno_ix]
+                    label = anno.label
+                    color = label_colors_dict[label]
+                    spans.append(f'<span class="annotation" style="color:{color}" data-label="{label}">')
+            prev_anno_ix = cur_anno_ix
+
+            if cs.partition_key() == current_consecutive_chars[0].partition_key():
+                # If the current charchter is in the same "group" as the previous one
+                current_consecutive_chars.append(cs)
+            else:
+                # Otherwise we make a span for the previous group
+                spans.append(
+                    EncodingVisualizer.consecutive_chars_to_html(
+                        current_consecutive_chars,
+                        text=text,
+                        encoding=encoding,
+                    )
+                )
+                # An reset the consecutive_char_list to form a new group
+                current_consecutive_chars = [cs]
+        # All that's left is to fill out the final span
+        # TODO I think there is an edge case here where an annotation's span might not close
+        spans.append(
+            EncodingVisualizer.consecutive_chars_to_html(
+                current_consecutive_chars,
+                text=text,
+                encoding=encoding,
+            )
+        )
+        res = HTMLBody(spans)  # Send the list of spans to the body of our html
+        return res
+
+    @staticmethod
+    def __make_anno_map(text: str, annotations: AnnotationList) -> PartialIntList:
+        """
+        Args:
+            text (:obj:`str`):
+                The raw text we want to align to
+
+            annotations (:obj:`AnnotationList`):
+                A (possibly empty) list of annotations
+
+        Returns:
+            A list of  length len(text) whose entry at index i is None if there is no annotation on
+            character i or k, the index of the annotation that covers index i where k is with
+            respect to the list of annotations
+        """
+        annotation_map = [None] * len(text)
+        for anno_ix, a in enumerate(annotations):
+            for i in range(a.start, a.end):
+                annotation_map[i] = anno_ix
+        return annotation_map
+
+    @staticmethod
+    def __make_char_states(text: str, encoding: Encoding, annotations: AnnotationList) -> List[CharState]:
+        """
+        For each character in the original text, we emit a tuple representing it's "state":
+
+            * which token_ix it corresponds to
+            * which word_ix it corresponds to
+            * which annotation_ix it corresponds to
+
+        Args:
+            text (:obj:`str`):
+                The raw text we want to align to
+
+            annotations (:obj:`List[Annotation]`):
+                A (possibly empty) list of annotations
+
+            encoding: (:class:`~tokenizers.Encoding`):
+                The encoding returned from the tokenizer
+
+        Returns:
+            :obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what
+            it's state is
+        """
+        annotation_map = EncodingVisualizer.__make_anno_map(text, annotations)
+        # Todo make this a dataclass or named tuple
+        char_states: List[CharState] = [CharState(char_ix) for char_ix in range(len(text))]
+        for token_ix, token in enumerate(encoding.tokens):
+            offsets = encoding.token_to_chars(token_ix)
+            if offsets is not None:
+                start, end = offsets
+                for i in range(start, end):
+                    char_states[i].tokens.append(token_ix)
+        for char_ix, anno_ix in enumerate(annotation_map):
+            char_states[char_ix].anno_ix = anno_ix
+
+        return char_states
+
+
+def HTMLBody(children: List[str], css_styles=css) -> str:
+    """
+    Generates the full html with css from a list of html spans
+
+    Args:
+        children (:obj:`List[str]`):
+            A list of strings, assumed to be html elements
+
+        css_styles (:obj:`str`, `optional`):
+            Optional alternative implementation of the css
+
+    Returns:
+        :obj:`str`: An HTML string with style markup
+    """
+    children_text = "".join(children)
+    return f"""
+    <html>
+        <head>
+            <style>
+                {css_styles}
+            </style>
+        </head>
+        <body>
+            <div class="tokenized-text" dir=auto>
+            {children_text}
+            </div>
+        </body>
+    </html>
+    """

source/tokenizers/trainers/__init__.py

@@ -0,0 +1,8 @@
+# Generated content DO NOT EDIT
+from .. import trainers
+
+Trainer = trainers.Trainer
+BpeTrainer = trainers.BpeTrainer
+UnigramTrainer = trainers.UnigramTrainer
+WordLevelTrainer = trainers.WordLevelTrainer
+WordPieceTrainer = trainers.WordPieceTrainer

source/tokenizers/trainers/__init__.pyi

@@ -0,0 +1,462 @@
+# Generated content DO NOT EDIT
+class Trainer:
+    """
+    Base class for all trainers
+
+    This class is not supposed to be instantiated directly. Instead, any implementation of a
+    Trainer will return an instance of this class when instantiated.
+    """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+class BpeTrainer(Trainer):
+    """
+    Trainer capable of training a BPE model
+
+    Args:
+        vocab_size (:obj:`int`, `optional`):
+            The size of the final vocabulary, including all tokens and alphabet.
+
+        min_frequency (:obj:`int`, `optional`):
+            The minimum frequency a pair should have in order to be merged.
+
+        show_progress (:obj:`bool`, `optional`):
+            Whether to show progress bars while training.
+
+        special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
+            A list of special tokens the model should know of.
+
+        limit_alphabet (:obj:`int`, `optional`):
+            The maximum different characters to keep in the alphabet.
+
+        initial_alphabet (:obj:`List[str]`, `optional`):
+            A list of characters to include in the initial alphabet, even
+            if not seen in the training dataset.
+            If the strings contain more than one character, only the first one
+            is kept.
+
+        continuing_subword_prefix (:obj:`str`, `optional`):
+            A prefix to be used for every subword that is not a beginning-of-word.
+
+        end_of_word_suffix (:obj:`str`, `optional`):
+            A suffix to be used for every subword that is a end-of-word.
+
+        max_token_length (:obj:`int`, `optional`):
+            Prevents creating tokens longer than the specified size.
+            This can help with reducing polluting your vocabulary with
+            highly repetitive tokens like `======` for wikipedia
+
+    """
+    def __init__(
+        self,
+        vocab_size=30000,
+        min_frequency=0,
+        show_progress=True,
+        special_tokens=[],
+        limit_alphabet=None,
+        initial_alphabet=[],
+        continuing_subword_prefix=None,
+        end_of_word_suffix=None,
+        max_token_length=None,
+        words={},
+    ):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def continuing_subword_prefix(self):
+        """ """
+        pass
+
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, value):
+        """ """
+        pass
+
+    @property
+    def end_of_word_suffix(self):
+        """ """
+        pass
+
+    @end_of_word_suffix.setter
+    def end_of_word_suffix(self, value):
+        """ """
+        pass
+
+    @property
+    def initial_alphabet(self):
+        """ """
+        pass
+
+    @initial_alphabet.setter
+    def initial_alphabet(self, value):
+        """ """
+        pass
+
+    @property
+    def limit_alphabet(self):
+        """ """
+        pass
+
+    @limit_alphabet.setter
+    def limit_alphabet(self, value):
+        """ """
+        pass
+
+    @property
+    def max_token_length(self):
+        """ """
+        pass
+
+    @max_token_length.setter
+    def max_token_length(self, value):
+        """ """
+        pass
+
+    @property
+    def min_frequency(self):
+        """ """
+        pass
+
+    @min_frequency.setter
+    def min_frequency(self, value):
+        """ """
+        pass
+
+    @property
+    def show_progress(self):
+        """ """
+        pass
+
+    @show_progress.setter
+    def show_progress(self, value):
+        """ """
+        pass
+
+    @property
+    def special_tokens(self):
+        """ """
+        pass
+
+    @special_tokens.setter
+    def special_tokens(self, value):
+        """ """
+        pass
+
+    @property
+    def vocab_size(self):
+        """ """
+        pass
+
+    @vocab_size.setter
+    def vocab_size(self, value):
+        """ """
+        pass
+
+class UnigramTrainer(Trainer):
+    """
+    Trainer capable of training a Unigram model
+
+    Args:
+        vocab_size (:obj:`int`):
+            The size of the final vocabulary, including all tokens and alphabet.
+
+        show_progress (:obj:`bool`):
+            Whether to show progress bars while training.
+
+        special_tokens (:obj:`List[Union[str, AddedToken]]`):
+            A list of special tokens the model should know of.
+
+        initial_alphabet (:obj:`List[str]`):
+            A list of characters to include in the initial alphabet, even
+            if not seen in the training dataset.
+            If the strings contain more than one character, only the first one
+            is kept.
+
+        shrinking_factor (:obj:`float`):
+            The shrinking factor used at each step of the training to prune the
+            vocabulary.
+
+        unk_token (:obj:`str`):
+            The token used for out-of-vocabulary tokens.
+
+        max_piece_length (:obj:`int`):
+            The maximum length of a given token.
+
+        n_sub_iterations (:obj:`int`):
+            The number of iterations of the EM algorithm to perform before
+            pruning the vocabulary.
+    """
+    def __init__(
+        self,
+        vocab_size=8000,
+        show_progress=True,
+        special_tokens=[],
+        initial_alphabet=[],
+        shrinking_factor=0.75,
+        unk_token=None,
+        max_piece_length=16,
+        n_sub_iterations=2,
+    ):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def initial_alphabet(self):
+        """ """
+        pass
+
+    @initial_alphabet.setter
+    def initial_alphabet(self, value):
+        """ """
+        pass
+
+    @property
+    def show_progress(self):
+        """ """
+        pass
+
+    @show_progress.setter
+    def show_progress(self, value):
+        """ """
+        pass
+
+    @property
+    def special_tokens(self):
+        """ """
+        pass
+
+    @special_tokens.setter
+    def special_tokens(self, value):
+        """ """
+        pass
+
+    @property
+    def vocab_size(self):
+        """ """
+        pass
+
+    @vocab_size.setter
+    def vocab_size(self, value):
+        """ """
+        pass
+
+class WordLevelTrainer(Trainer):
+    """
+    Trainer capable of training a WorldLevel model
+
+    Args:
+        vocab_size (:obj:`int`, `optional`):
+            The size of the final vocabulary, including all tokens and alphabet.
+
+        min_frequency (:obj:`int`, `optional`):
+            The minimum frequency a pair should have in order to be merged.
+
+        show_progress (:obj:`bool`, `optional`):
+            Whether to show progress bars while training.
+
+        special_tokens (:obj:`List[Union[str, AddedToken]]`):
+            A list of special tokens the model should know of.
+    """
+    def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[]):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def min_frequency(self):
+        """ """
+        pass
+
+    @min_frequency.setter
+    def min_frequency(self, value):
+        """ """
+        pass
+
+    @property
+    def show_progress(self):
+        """ """
+        pass
+
+    @show_progress.setter
+    def show_progress(self, value):
+        """ """
+        pass
+
+    @property
+    def special_tokens(self):
+        """ """
+        pass
+
+    @special_tokens.setter
+    def special_tokens(self, value):
+        """ """
+        pass
+
+    @property
+    def vocab_size(self):
+        """ """
+        pass
+
+    @vocab_size.setter
+    def vocab_size(self, value):
+        """ """
+        pass
+
+class WordPieceTrainer(Trainer):
+    """
+    Trainer capable of training a WordPiece model
+
+    Args:
+        vocab_size (:obj:`int`, `optional`):
+            The size of the final vocabulary, including all tokens and alphabet.
+
+        min_frequency (:obj:`int`, `optional`):
+            The minimum frequency a pair should have in order to be merged.
+
+        show_progress (:obj:`bool`, `optional`):
+            Whether to show progress bars while training.
+
+        special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`):
+            A list of special tokens the model should know of.
+
+        limit_alphabet (:obj:`int`, `optional`):
+            The maximum different characters to keep in the alphabet.
+
+        initial_alphabet (:obj:`List[str]`, `optional`):
+            A list of characters to include in the initial alphabet, even
+            if not seen in the training dataset.
+            If the strings contain more than one character, only the first one
+            is kept.
+
+        continuing_subword_prefix (:obj:`str`, `optional`):
+            A prefix to be used for every subword that is not a beginning-of-word.
+
+        end_of_word_suffix (:obj:`str`, `optional`):
+            A suffix to be used for every subword that is a end-of-word.
+    """
+    def __init__(
+        self,
+        vocab_size=30000,
+        min_frequency=0,
+        show_progress=True,
+        special_tokens=[],
+        limit_alphabet=None,
+        initial_alphabet=[],
+        continuing_subword_prefix="##",
+        end_of_word_suffix=None,
+    ):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def continuing_subword_prefix(self):
+        """ """
+        pass
+
+    @continuing_subword_prefix.setter
+    def continuing_subword_prefix(self, value):
+        """ """
+        pass
+
+    @property
+    def end_of_word_suffix(self):
+        """ """
+        pass
+
+    @end_of_word_suffix.setter
+    def end_of_word_suffix(self, value):
+        """ """
+        pass
+
+    @property
+    def initial_alphabet(self):
+        """ """
+        pass
+
+    @initial_alphabet.setter
+    def initial_alphabet(self, value):
+        """ """
+        pass
+
+    @property
+    def limit_alphabet(self):
+        """ """
+        pass
+
+    @limit_alphabet.setter
+    def limit_alphabet(self, value):
+        """ """
+        pass
+
+    @property
+    def min_frequency(self):
+        """ """
+        pass
+
+    @min_frequency.setter
+    def min_frequency(self, value):
+        """ """
+        pass
+
+    @property
+    def show_progress(self):
+        """ """
+        pass
+
+    @show_progress.setter
+    def show_progress(self, value):
+        """ """
+        pass
+
+    @property
+    def special_tokens(self):
+        """ """
+        pass
+
+    @special_tokens.setter
+    def special_tokens(self, value):
+        """ """
+        pass
+
+    @property
+    def vocab_size(self):
+        """ """
+        pass
+
+    @vocab_size.setter
+    def vocab_size(self, value):
+        """ """
+        pass

source/torchaudio-2.9.1.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/torchaudio-2.9.1.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: torchaudio
+Version: 2.9.1
+Summary: An audio package for PyTorch
+Home-page: https://github.com/pytorch/audio
+Author: Soumith Chintala, David Pollack, Sean Naren, Peter Goldsborough, Moto Hira, Caroline Chen, Jeff Hwang, Zhaoheng Ni, Xiaohui Zhang
+Author-email: soumith@pytorch.org
+Maintainer: Moto Hira, Caroline Chen, Jeff Hwang, Zhaoheng Ni, Xiaohui Zhang
+Maintainer-email: moto@meta.com
+Classifier: Environment :: Plugins
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch==2.9.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: maintainer
+Dynamic: maintainer-email
+Dynamic: requires-dist
+Dynamic: summary
+
+torchaudio: an audio library for PyTorch
+========================================
+
+[![Documentation](https://img.shields.io/badge/dynamic/json.svg?label=docs&url=https%3A%2F%2Fpypi.org%2Fpypi%2Ftorchaudio%2Fjson&query=%24.info.version&colorB=brightgreen&prefix=v)](https://pytorch.org/audio/main/)
+[![Anaconda Badge](https://anaconda.org/pytorch/torchaudio/badges/downloads.svg)](https://anaconda.org/pytorch/torchaudio)
+[![Anaconda-Server Badge](https://anaconda.org/pytorch/torchaudio/badges/platforms.svg)](https://anaconda.org/pytorch/torchaudio)
+
+![TorchAudio Logo](docs/source/_static/img/logo.png)
+
+> [!NOTE]
+> **We have transitioned TorchAudio into a
+>  maintenance phase. This process removed some user-facing
+>  features. These features were deprecated from TorchAudio 2.8 and removed in 2.9.
+>  Our main goals were to reduce redundancies with the rest of the
+>  PyTorch ecosystem, make it easier to maintain, and create a version of
+>  TorchAudio that is more tightly scoped to its strengths: processing audio
+>  data for ML. Please see
+>  [our community message](https://github.com/pytorch/audio/issues/3902)
+>  for more details.**
+
+The aim of torchaudio is to apply [PyTorch](https://github.com/pytorch/pytorch) to
+the audio domain. By supporting PyTorch, torchaudio follows the same philosophy
+of providing strong GPU acceleration, having a focus on trainable features through
+the autograd system, and having consistent style (tensor names and dimension names).
+Therefore, it is primarily a machine learning library and not a general signal
+processing library. The benefits of PyTorch can be seen in torchaudio through
+having all the computations be through PyTorch operations which makes it easy
+to use and feel like a natural extension.
+
+- [Dataloaders for common audio datasets](http://pytorch.org/audio/main/datasets.html)
+- Audio and speech processing functions
+  - [forced_align](https://pytorch.org/audio/main/generated/torchaudio.functional.forced_align.html)
+- Common audio transforms
+  - [Spectrogram, AmplitudeToDB, MelScale, MelSpectrogram, MFCC, MuLawEncoding, MuLawDecoding, Resample](http://pytorch.org/audio/main/transforms.html)
+- Compliance interfaces: Run code using PyTorch that align with other libraries
+  - [Kaldi: spectrogram, fbank, mfcc](https://pytorch.org/audio/main/compliance.kaldi.html)
+
+Installation
+------------
+
+Please refer to https://pytorch.org/audio/main/installation.html for installation and build process of TorchAudio.
+
+
+API Reference
+-------------
+
+API Reference is located here: http://pytorch.org/audio/main/
+
+Contributing Guidelines
+-----------------------
+
+Please refer to [CONTRIBUTING.md](./CONTRIBUTING.md)
+
+Citation
+--------
+
+If you find this package useful, please cite as:
+
+```bibtex
+@article{yang2021torchaudio,
+  title={TorchAudio: Building Blocks for Audio and Speech Processing},
+  author={Yao-Yuan Yang and Moto Hira and Zhaoheng Ni and Anjali Chourdia and Artyom Astafurov and Caroline Chen and Ching-Feng Yeh and Christian Puhrsch and David Pollack and Dmitriy Genzel and Donny Greenberg and Edward Z. Yang and Jason Lian and Jay Mahadeokar and Jeff Hwang and Ji Chen and Peter Goldsborough and Prabhat Roy and Sean Narenthiran and Shinji Watanabe and Soumith Chintala and Vincent Quenneville-Bélair and Yangyang Shi},
+  journal={arXiv preprint arXiv:2110.15018},
+  year={2021}
+}
+```
+
+```bibtex
+@misc{hwang2023torchaudio,
+      title={TorchAudio 2.1: Advancing speech recognition, self-supervised learning, and audio processing components for PyTorch},
+      author={Jeff Hwang and Moto Hira and Caroline Chen and Xiaohui Zhang and Zhaoheng Ni and Guangzhi Sun and Pingchuan Ma and Ruizhe Huang and Vineel Pratap and Yuekai Zhang and Anurag Kumar and Chin-Yun Yu and Chuang Zhu and Chunxi Liu and Jacob Kahn and Mirco Ravanelli and Peng Sun and Shinji Watanabe and Yangyang Shi and Yumeng Tao and Robin Scheibler and Samuele Cornell and Sean Kim and Stavros Petridis},
+      year={2023},
+      eprint={2310.17864},
+      archivePrefix={arXiv},
+      primaryClass={eess.AS}
+}
+```
+
+Disclaimer on Datasets
+----------------------
+
+This is a utility library that downloads and prepares public datasets. We do not host or distribute these datasets, vouch for their quality or fairness, or claim that you have license to use the dataset. It is your responsibility to determine whether you have permission to use the dataset under the dataset's license.
+
+If you're a dataset owner and wish to update any part of it (description, citation, etc.), or do not want your dataset to be included in this library, please get in touch through a GitHub issue. Thanks for your contribution to the ML community!
+
+Pre-trained Model License
+-------------------------
+
+The pre-trained models provided in this library may have their own licenses or terms and conditions derived from the dataset used for training. It is your responsibility to determine whether you have permission to use the models for your use case.
+
+For instance, SquimSubjective model is released under the Creative Commons Attribution Non Commercial 4.0 International (CC-BY-NC 4.0) license. See [the link](https://zenodo.org/record/4660670#.ZBtWPOxuerN) for additional details.
+
+Other pre-trained models that have different license are noted in documentation. Please checkout the [documentation page](https://pytorch.org/audio/main/).

source/torchaudio-2.9.1.dist-info/RECORD

@@ -0,0 +1,166 @@
+torchaudio-2.9.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+torchaudio-2.9.1.dist-info/METADATA,sha256=nhbW4EcMPskMVtj653CC9bZ2xsogo4xjD7mkmp7K9Sg,6911
+torchaudio-2.9.1.dist-info/RECORD,,
+torchaudio-2.9.1.dist-info/WHEEL,sha256=VXvNKn6nFeCM45GEUrNLJOO_J_e-cNJphGt9rWFxyE0,113
+torchaudio-2.9.1.dist-info/licenses/LICENSE,sha256=k6WIYahYzBCOa2uDPgjnbosqZjOeSoAHyKWowf-cQNY,1338
+torchaudio-2.9.1.dist-info/top_level.txt,sha256=mPKWMIRWWW2JwbJN6wRckeN1gpbjhifapAF0Z9t7SMo,11
+torchaudio/__init__.py,sha256=8OB3EPGCViF7LgBWy_bUyZUF6HJUIpbTI8ouRGwn6lU,7878
+torchaudio/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/__pycache__/_torchcodec.cpython-312.pyc,,
+torchaudio/__pycache__/version.cpython-312.pyc,,
+torchaudio/_extension/__init__.py,sha256=A8oH7eF2Fx4d68LddkFE1Ylq3AE3X2sgZdXjvaMEdjQ,1905
+torchaudio/_extension/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/_extension/__pycache__/utils.cpython-312.pyc,,
+torchaudio/_extension/utils.py,sha256=UQCObmKAsgdHhXU2dQYYxyFXwfdTsBO9bnrQmpQNN_I,4926
+torchaudio/_internal/__init__.py,sha256=gjU8g9HhVd9hHrHXJM0xOlZL6cT8ktO60MN8RHI6ZbA,241
+torchaudio/_internal/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/_internal/__pycache__/module_utils.cpython-312.pyc,,
+torchaudio/_internal/module_utils.py,sha256=eosQSGtN5WhHhATJGBWJIGUM_nvtgLPRkQ8BH_Zd53o,5229
+torchaudio/_torchcodec.py,sha256=Z1TpONctbL80DufuWhLRj4dC0rVhjKu6hOYeglcLwvU,13424
+torchaudio/compliance/__init__.py,sha256=hhNObUS0c-fS-VMudM7zl3-CvupvCDmESlikntSMn5g,48
+torchaudio/compliance/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/compliance/__pycache__/kaldi.cpython-312.pyc,,
+torchaudio/compliance/kaldi.py,sha256=XL6hpYTd6nSPb2imIdeU4TM06I2fqh1AmG968y8ZbSk,36666
+torchaudio/datasets/__init__.py,sha256=taRr3duDaEK1Pfzj9N1dFuZpXfy8e4uFItcJiRLAQwQ,1171
+torchaudio/datasets/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/cmuarctic.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/cmudict.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/commonvoice.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/dr_vctk.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/fluentcommands.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/gtzan.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/iemocap.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/librilight_limited.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/librimix.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/librispeech.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/librispeech_biasing.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/libritts.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/ljspeech.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/musdb_hq.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/quesst14.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/snips.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/speechcommands.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/tedlium.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/utils.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/vctk.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/voxceleb1.cpython-312.pyc,,
+torchaudio/datasets/__pycache__/yesno.cpython-312.pyc,,
+torchaudio/datasets/cmuarctic.py,sha256=2e5Oh_jDHRs8ORhNONsD9NhI_OfQSHDLQAM-tWpgZ-U,7081
+torchaudio/datasets/cmudict.py,sha256=9OEpNDYpyqeEyinAnyGIU8FampDj7ziSOHRwJLIlq2M,5990
+torchaudio/datasets/commonvoice.py,sha256=9khedUCmdEkCKPU6_r8VWz6I2VdJokatuziZ6BxJMZs,2763
+torchaudio/datasets/dr_vctk.py,sha256=Km4-tKllAgnOKCuq66YRWhTlNWmC7D0Xz3dAttRRGSo,4377
+torchaudio/datasets/fluentcommands.py,sha256=u3tkO4-AAaTWdbRQi6lIvad4x2plZgXM39KljGtmRsw,3245
+torchaudio/datasets/gtzan.py,sha256=I5dRP_QGuQ1joXWRwZwtvpwi22uZTb8QZm9Mr2W55Mg,24357
+torchaudio/datasets/iemocap.py,sha256=X_WCoXOzRqcWRRRoUtY0AlD9SJcUUOACIcgbV0irt48,4930
+torchaudio/datasets/librilight_limited.py,sha256=fAwpX0hEMze5aV57BP7rjBLwRiZa3Aje_NXi_3o16wA,4179
+torchaudio/datasets/librimix.py,sha256=VtKOhf6VJc1ysWCvUvh0SbtjOkXJChmBM_BhoSkg_2A,5116
+torchaudio/datasets/librispeech.py,sha256=zkzJFWchWs4AktYAI-ghmWH4ZeJ84C0uDo9E1_pTgSI,6308
+torchaudio/datasets/librispeech_biasing.py,sha256=d-02tyrXI-CSGbXBFYFcnM_yT8WSGABHfpNiFxyadL0,6958
+torchaudio/datasets/libritts.py,sha256=EtWOoCDz7_qGLZF5YcZfnHaLxH4Y8QJCnopafLiqFno,5870
+torchaudio/datasets/ljspeech.py,sha256=92NeLQsC1iKpqfiMkKKbcJDpaYdZKVdVEBQJze1wmxY,3494
+torchaudio/datasets/musdb_hq.py,sha256=TYKjpat6JKr9bkFqUecu7_hRdshRfQP2UbknaYR3Q0U,5075
+torchaudio/datasets/quesst14.py,sha256=QyGd4fMS820ATbP8YgBtu7bSSK09pw5RZklsPJ8Jf0Y,4455
+torchaudio/datasets/snips.py,sha256=WaYUknGFM3rnLklOj5ZYHSX5mhlf_Ce4p3LBZdA9yJc,5008
+torchaudio/datasets/speechcommands.py,sha256=cLSgiVYlQjEOuYPpFeAtcXSGirraH4IMoP8p9WIvUoY,7481
+torchaudio/datasets/tedlium.py,sha256=a8Hf2QvOki7_chgXcMAFMk-piTjodktfnc3HRbUVJkU,8698
+torchaudio/datasets/utils.py,sha256=P6nckh2YrAfOPMphHlxyfI-HBmNg39DTlxQ8-asG4MY,1703
+torchaudio/datasets/vctk.py,sha256=twR_n8LyQcT8A_HrJoMx3RkaVrRXXZAnIVU1d0E0npQ,5699
+torchaudio/datasets/voxceleb1.py,sha256=9vU0ftB4-2usO8ZiEUKR_IQTEdHhA0M8l9scXCNehnw,11725
+torchaudio/datasets/yesno.py,sha256=4sgfMeSxz8HaRDk6A2UIFP-20q29MwEO_r8DoEtfbvE,3026
+torchaudio/functional/__init__.py,sha256=_5eT3FZFO6GXmKqFkPY4c_w7F7Isqnd8CTP2FdMxfVM,2451
+torchaudio/functional/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/functional/__pycache__/_alignment.cpython-312.pyc,,
+torchaudio/functional/__pycache__/filtering.cpython-312.pyc,,
+torchaudio/functional/__pycache__/functional.cpython-312.pyc,,
+torchaudio/functional/_alignment.py,sha256=NveQ74x8PmleuB-Ka9eEYYyshbV7nYc0g-Tu3NGHdz0,4739
+torchaudio/functional/filtering.py,sha256=rML8MismfehSeglw65kUkfugoP6XDtWcs_XhCl6aJM4,62325
+torchaudio/functional/functional.py,sha256=5l-07BLVAs1PNU8NM2CPV_GTnq3V8nbV9tI7t0v79Y4,94731
+torchaudio/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+torchaudio/lib/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/lib/_torchaudio.so,sha256=70IIA3F6xzE4xCzUrpWAGTfHu1KqWzIM1KxU4dDTTsg,171552
+torchaudio/lib/libctc_prefix_decoder.so,sha256=1DlwTtNZXE_P0zsHvoVP7mBzpSpUJFaSSaaQjg8oX0E,6268592
+torchaudio/lib/libtorchaudio.so,sha256=L8j8al4FAtxyb5w5Arp5eo-jpVlM2HcswimRW7c2mII,2573624
+torchaudio/lib/pybind11_prefixctc.so,sha256=VdBdtmt8GU2y1ulK-S4oLR5jWYA5K0PnlEBSqrV4F3A,277688
+torchaudio/models/__init__.py,sha256=BNMNGuwpJAFRsdtwHYQ6slGClkrUTu31_7mXh7FjeV4,1995
+torchaudio/models/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/models/__pycache__/_hdemucs.cpython-312.pyc,,
+torchaudio/models/__pycache__/conformer.cpython-312.pyc,,
+torchaudio/models/__pycache__/conv_tasnet.cpython-312.pyc,,
+torchaudio/models/__pycache__/deepspeech.cpython-312.pyc,,
+torchaudio/models/__pycache__/emformer.cpython-312.pyc,,
+torchaudio/models/__pycache__/rnnt.cpython-312.pyc,,
+torchaudio/models/__pycache__/rnnt_decoder.cpython-312.pyc,,
+torchaudio/models/__pycache__/tacotron2.cpython-312.pyc,,
+torchaudio/models/__pycache__/wav2letter.cpython-312.pyc,,
+torchaudio/models/__pycache__/wavernn.cpython-312.pyc,,
+torchaudio/models/_hdemucs.py,sha256=VPnQ73lA9lfAxRjZ85NCGJYP36mPNwTjS-TU4qelu_k,38242
+torchaudio/models/conformer.py,sha256=5IceU-jcZKofkHTTqRKoytubQ75MzZPrPlfkLsIlxeA,10068
+torchaudio/models/conv_tasnet.py,sha256=v-DI_Ej9FCBBbSH-Spkh3tzq8rkBhbQNA-Wp52Uf32E,12540
+torchaudio/models/decoder/__init__.py,sha256=HxU2Bgyea0No8SORRfxgMZNwwEDTrjlT3bDW_GxzpTU,1899
+torchaudio/models/decoder/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/models/decoder/__pycache__/_ctc_decoder.cpython-312.pyc,,
+torchaudio/models/decoder/__pycache__/_cuda_ctc_decoder.cpython-312.pyc,,
+torchaudio/models/decoder/_ctc_decoder.py,sha256=AmLQAcm4Q4bFPqnq-SF7Lpvg2QPK88xyio8ol_OJjvU,20086
+torchaudio/models/decoder/_cuda_ctc_decoder.py,sha256=xFrj1cTEsS-MxAO5Vgdutcb3kTb7Jv-OFhS6cmfFKhA,7186
+torchaudio/models/deepspeech.py,sha256=kQW3B6YcjYuq7xRzWjRJFGr7ZNraY9gMYDTxII7Cgtg,2746
+torchaudio/models/emformer.py,sha256=ncDeEcYegUmIKQoDBoufUhVWj4dYpZAXxLX0qmEqt1A,37766
+torchaudio/models/rnnt.py,sha256=jz66nwDd1qGT6KQR1lbA_urPktygewhm0FH66T7P3Ek,35541
+torchaudio/models/rnnt_decoder.py,sha256=IwlDsuw1SA-uCRrXGMBqm05auGFSha2bZ-8BOImnK0c,12839
+torchaudio/models/squim/__init__.py,sha256=b98nAaL28Q4w3lrqd_6wUd0An-xNhhJn4Tj8oZlzQnc,346
+torchaudio/models/squim/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/models/squim/__pycache__/objective.cpython-312.pyc,,
+torchaudio/models/squim/__pycache__/subjective.cpython-312.pyc,,
+torchaudio/models/squim/objective.py,sha256=gvUasz7RpqgKeGf04yHUotshSIzH3KzjW90-iHeDo2g,12281
+torchaudio/models/squim/subjective.py,sha256=N00kILSPm0akWyNsrNYKmHgZmooo8gbyUm5IVLf7bx8,5797
+torchaudio/models/tacotron2.py,sha256=FimYhGSI8FKwWb87CLk4h3yKWatCU2HvFmU1t5WUn4E,45914
+torchaudio/models/wav2letter.py,sha256=KNcq4p0qZG2Bwfdakv7YwLCvi_yGT-qB4fJwGMuFQhg,3278
+torchaudio/models/wav2vec2/__init__.py,sha256=WlafukV6GwuSNh0CZifrYUt4V5l59kjvGX7AZNonjfk,927
+torchaudio/models/wav2vec2/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/models/wav2vec2/__pycache__/components.cpython-312.pyc,,
+torchaudio/models/wav2vec2/__pycache__/model.cpython-312.pyc,,
+torchaudio/models/wav2vec2/__pycache__/wavlm_attention.cpython-312.pyc,,
+torchaudio/models/wav2vec2/components.py,sha256=DRmW-GHYf-JReCg_0l1ovNWJBnAavePO3S2vPY-1ze4,47077
+torchaudio/models/wav2vec2/model.py,sha256=Z2VN6KbDOOdq5JtP7lxPQebwYqsxKms1Eu4IjDJtZaQ,60092
+torchaudio/models/wav2vec2/utils/__init__.py,sha256=qmMbz4HAN5kEEyl4cSGm_JQZI47beyh4witydPC_qns,181
+torchaudio/models/wav2vec2/utils/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/models/wav2vec2/utils/__pycache__/import_fairseq.cpython-312.pyc,,
+torchaudio/models/wav2vec2/utils/__pycache__/import_huggingface.cpython-312.pyc,,
+torchaudio/models/wav2vec2/utils/import_fairseq.py,sha256=oCwG6qpG0bCXue2V56fjDcC8cA2rgy4b3O_nu_FI9ZY,9198
+torchaudio/models/wav2vec2/utils/import_huggingface.py,sha256=1nVCipp-lOUAyl_-P103DWLUeTOZi9X_ffX93bOXxEk,5946
+torchaudio/models/wav2vec2/wavlm_attention.py,sha256=1DU_pkoLCeHQwSF4lJ06cez0PsMVoXNxiYKP0Yv0qFQ,10844
+torchaudio/models/wavernn.py,sha256=5xUyao5g69jRXX4ReNi4mP_aTSIonJPP6XcPrqKybEk,15446
+torchaudio/pipelines/__init__.py,sha256=Xy8NmInKwTcNBHwLTTjHjrfczRLuQq8a67ENt1OTVXM,2745
+torchaudio/pipelines/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/pipelines/__pycache__/_source_separation_pipeline.cpython-312.pyc,,
+torchaudio/pipelines/__pycache__/_squim_pipeline.cpython-312.pyc,,
+torchaudio/pipelines/__pycache__/rnnt_pipeline.cpython-312.pyc,,
+torchaudio/pipelines/_source_separation_pipeline.py,sha256=ogWakvaOv6OegmREcbagvfIm0jNWjzEtsdMYTialRNk,4225
+torchaudio/pipelines/_squim_pipeline.py,sha256=852SYXqUZDgTPegL7LqgVQr0PXG94da_DTDF2bwDhVE,6282
+torchaudio/pipelines/_tts/__init__.py,sha256=PP7l8XzVURqelwuMJFgfOCv4fvzZunDiy90ZQlRkv7g,426
+torchaudio/pipelines/_tts/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/pipelines/_tts/__pycache__/impl.cpython-312.pyc,,
+torchaudio/pipelines/_tts/__pycache__/interface.cpython-312.pyc,,
+torchaudio/pipelines/_tts/__pycache__/utils.cpython-312.pyc,,
+torchaudio/pipelines/_tts/impl.py,sha256=Tig4_5sITJADwxN5eZGek7Ath_-e3sV8CTM5t6UpeUU,15374
+torchaudio/pipelines/_tts/interface.py,sha256=yUaS0UK3PTRruYXRWFil7lAhr-1iYiyBaDBLmEnJPUQ,10224
+torchaudio/pipelines/_tts/utils.py,sha256=KGrFoetCZ4l4FJkINFptAc8Pvrbo9e4QQhCIMCp8NYY,4810
+torchaudio/pipelines/_wav2vec2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+torchaudio/pipelines/_wav2vec2/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/pipelines/_wav2vec2/__pycache__/aligner.cpython-312.pyc,,
+torchaudio/pipelines/_wav2vec2/__pycache__/impl.cpython-312.pyc,,
+torchaudio/pipelines/_wav2vec2/__pycache__/utils.cpython-312.pyc,,
+torchaudio/pipelines/_wav2vec2/aligner.py,sha256=pIWRgQ-kdYUxtL8bdc0qk9wBjwRrHY1uSWL3L4e2vxs,2709
+torchaudio/pipelines/_wav2vec2/impl.py,sha256=zdXFjytJO5MvnB-3aygzUUFKxCTkQGU_OX_rhUh9c0k,65561
+torchaudio/pipelines/_wav2vec2/utils.py,sha256=Q8_fWOR2JDnHu0TTRmHzRjI3BOJa0hGIAl0cjtALgsQ,6971
+torchaudio/pipelines/rnnt_pipeline.py,sha256=56nQnCcjY4xewDqXR1Rkrh_hyoK42CsYumpU8mUNs1w,13753
+torchaudio/transforms/__init__.py,sha256=8_47qPRjXNg332f2kcNP_T5UXCn6jQmUUMkIgyIByjY,1398
+torchaudio/transforms/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/transforms/__pycache__/_multi_channel.cpython-312.pyc,,
+torchaudio/transforms/__pycache__/_transforms.cpython-312.pyc,,
+torchaudio/transforms/_multi_channel.py,sha256=GZ2rrwFt2KtSG7At7kS9Bqh1KmYYw0HwcUnEjc-AWr8,22221
+torchaudio/transforms/_transforms.py,sha256=i-xEARqCfnaDk9b0yzmYkPo9Gg1N1iKvZiLSMdX14-Q,86919
+torchaudio/utils/__init__.py,sha256=adAdfYm9DJBC2JXxRCTrjxOUU1vKJ9w3rFke-DzKKqU,70
+torchaudio/utils/__pycache__/__init__.cpython-312.pyc,,
+torchaudio/utils/__pycache__/download.cpython-312.pyc,,
+torchaudio/utils/download.py,sha256=gZA7CijUoAu3Q0Qd6dKpFQAEjcdnxR6xOT59lTgEIOo,2883
+torchaudio/version.py,sha256=sBUsm0oAwNdEIPgVIrOs5KflkkUDSnUNB0usP957SGE,85

source/torchaudio-2.9.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: false
+Tag: cp312-cp312-manylinux_2_28_x86_64
+

source/torchaudio-2.9.1.dist-info/licenses/LICENSE

@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2017 Facebook Inc. (Soumith Chintala), 
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

source/torchaudio-2.9.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+torchaudio

source/torchaudio/__init__.py

@@ -0,0 +1,204 @@
+import os
+from typing import BinaryIO, Optional, Tuple, Union
+
+import torch
+
+# Initialize extension and backend first
+from . import _extension  # noqa  # usort: skip
+from . import compliance, datasets, functional, models, pipelines, transforms, utils  # noqa: F401
+from ._torchcodec import load_with_torchcodec, save_with_torchcodec
+
+
+try:
+    from .version import __version__, git_version  # noqa: F401
+except ImportError:
+    pass
+
+
+def load(
+    uri: Union[BinaryIO, str, os.PathLike],
+    frame_offset: int = 0,
+    num_frames: int = -1,
+    normalize: bool = True,
+    channels_first: bool = True,
+    format: Optional[str] = None,
+    buffer_size: int = 4096,
+    backend: Optional[str] = None,
+) -> Tuple[torch.Tensor, int]:
+    """Load audio data from source using TorchCodec's AudioDecoder.
+
+    .. note::
+
+        As of TorchAudio 2.9, this function relies on TorchCodec's decoding capabilities under the hood. It is
+        provided for convenience, but we do recommend that you port your code to
+        natively use ``torchcodec``'s ``AudioDecoder`` class for better
+        performance:
+        https://docs.pytorch.org/torchcodec/stable/generated/torchcodec.decoders.AudioDecoder.
+        Because of the reliance on Torchcodec, the parameters ``normalize``, ``buffer_size``, and
+        ``backend`` are ignored and accepted only for backwards compatibility.
+        To install torchcodec, follow the instructions at https://github.com/pytorch/torchcodec#installing-torchcodec.
+
+
+    Args:
+        uri (path-like object or file-like object):
+            Source of audio data. The following types are accepted:
+
+            * ``path-like``: File path or URL.
+            * ``file-like``: Object with ``read(size: int) -> bytes`` method.
+
+        frame_offset (int, optional):
+            Number of samples to skip before start reading data.
+        num_frames (int, optional):
+            Maximum number of samples to read. ``-1`` reads all the remaining samples,
+            starting from ``frame_offset``.
+        normalize (bool, optional):
+            TorchCodec always returns normalized float32 samples. This parameter
+            is ignored and a warning is issued if set to False.
+            Default: ``True``.
+        channels_first (bool, optional):
+            When True, the returned Tensor has dimension `[channel, time]`.
+            Otherwise, the returned Tensor's dimension is `[time, channel]`.
+        format (str or None, optional):
+            Format hint for the decoder. May not be supported by all TorchCodec
+            decoders. (Default: ``None``)
+        buffer_size (int, optional):
+            Not used by TorchCodec AudioDecoder. Provided for API compatibility.
+        backend (str or None, optional):
+            Not used by TorchCodec AudioDecoder. Provided for API compatibility.
+
+    Returns:
+        (torch.Tensor, int): Resulting Tensor and sample rate.
+        Always returns float32 tensors. If ``channels_first=True``, shape is
+        `[channel, time]`, otherwise `[time, channel]`.
+
+    Raises:
+        ImportError: If torchcodec is not available.
+        ValueError: If unsupported parameters are used.
+        RuntimeError: If TorchCodec fails to decode the audio.
+
+    Note:
+        - TorchCodec always returns normalized float32 samples, so the ``normalize``
+        parameter has no effect.
+        - The ``buffer_size`` and ``backend`` parameters are ignored.
+        - Not all audio formats supported by torchaudio backends may be supported
+        by TorchCodec.
+    """
+    return load_with_torchcodec(
+        uri,
+        frame_offset=frame_offset,
+        num_frames=num_frames,
+        normalize=normalize,
+        channels_first=channels_first,
+        format=format,
+        buffer_size=buffer_size,
+        backend=backend,
+    )
+
+
+def save(
+    uri: Union[str, os.PathLike],
+    src: torch.Tensor,
+    sample_rate: int,
+    channels_first: bool = True,
+    format: Optional[str] = None,
+    encoding: Optional[str] = None,
+    bits_per_sample: Optional[int] = None,
+    buffer_size: int = 4096,
+    backend: Optional[str] = None,
+    compression: Optional[Union[float, int]] = None,
+) -> None:
+    """Save audio data to file using TorchCodec's AudioEncoder.
+
+    .. note::
+
+        As of TorchAudio 2.9, this function relies on TorchCodec's encoding capabilities under the hood.
+        It is provided for convenience, but we do recommend that you port your code to
+        natively use ``torchcodec``'s ``AudioEncoder`` class for better
+        performance:
+        https://docs.pytorch.org/torchcodec/stable/generated/torchcodec.encoders.AudioEncoder.
+        Because of the reliance on Torchcodec, the parameters ``format``, ``encoding``,
+        ``bits_per_sample``, ``buffer_size``, and ``backend``, are ignored and accepted only for
+        backwards compatibility.
+        To install torchcodec, follow the instructions at https://github.com/pytorch/torchcodec#installing-torchcodec.
+
+    Args:
+        uri (path-like object):
+            Path to save the audio file. The file extension determines the format.
+
+        src (torch.Tensor):
+            Audio data to save. Must be a 1D or 2D tensor with float32 values
+            in the range [-1, 1]. If 2D, shape should be [channel, time] when
+            channels_first=True, or [time, channel] when channels_first=False.
+
+        sample_rate (int):
+            Sample rate of the audio data.
+
+        channels_first (bool, optional):
+            Indicates whether the input tensor has channels as the first dimension.
+            If True, expects [channel, time]. If False, expects [time, channel].
+            Default: True.
+
+        format (str or None, optional):
+            Audio format hint. Not used by TorchCodec (format is determined by
+            file extension). A warning is issued if provided.
+            Default: None.
+
+        encoding (str or None, optional):
+            Audio encoding. Not fully supported by TorchCodec AudioEncoder.
+            A warning is issued if provided. Default: None.
+
+        bits_per_sample (int or None, optional):
+            Bits per sample. Not directly supported by TorchCodec AudioEncoder.
+            A warning is issued if provided. Default: None.
+
+        buffer_size (int, optional):
+            Not used by TorchCodec AudioEncoder. Provided for API compatibility.
+            A warning is issued if not default value. Default: 4096.
+
+        backend (str or None, optional):
+            Not used by TorchCodec AudioEncoder. Provided for API compatibility.
+            A warning is issued if provided. Default: None.
+
+        compression (float, int or None, optional):
+            Compression level or bit rate. Maps to bit_rate parameter in
+            TorchCodec AudioEncoder. Default: None.
+
+    Raises:
+        ImportError: If torchcodec is not available.
+        ValueError: If input parameters are invalid.
+        RuntimeError: If TorchCodec fails to encode the audio.
+
+    Note:
+        - TorchCodec AudioEncoder expects float32 samples in [-1, 1] range.
+        - Some parameters (format, encoding, bits_per_sample, buffer_size, backend)
+        are not used by TorchCodec but are provided for API compatibility.
+        - The output format is determined by the file extension in the uri.
+        - TorchCodec uses FFmpeg under the hood for encoding.
+    """
+    return save_with_torchcodec(
+        uri,
+        src,
+        sample_rate,
+        channels_first=channels_first,
+        format=format,
+        encoding=encoding,
+        bits_per_sample=bits_per_sample,
+        buffer_size=buffer_size,
+        backend=backend,
+        compression=compression,
+    )
+
+
+__all__ = [
+    "load",
+    "load_with_torchcodec",
+    "save_with_torchcodec",
+    "save",
+    "compliance",
+    "datasets",
+    "functional",
+    "models",
+    "pipelines",
+    "utils",
+    "transforms",
+]

source/torchaudio/_extension/__init__.py

@@ -0,0 +1,61 @@
+import logging
+import os
+import sys
+
+from torchaudio._internal.module_utils import fail_with_message, is_module_available, no_op
+
+from .utils import _check_cuda_version, _init_dll_path, _load_lib
+
+_LG = logging.getLogger(__name__)
+
+
+# Note:
+# `_check_cuda_version` is not meant to be used by regular users.
+# Builder uses it for debugging purpose, so we export it.
+# https://github.com/pytorch/builder/blob/e2e4542b8eb0bdf491214451a1a4128bd606cce2/test/smoke_test/smoke_test.py#L80
+__all__ = [
+    "_check_cuda_version",
+    "_IS_TORCHAUDIO_EXT_AVAILABLE",
+    "_IS_RIR_AVAILABLE",
+]
+
+
+if os.name == "nt" and (3, 8) <= sys.version_info < (3, 9):
+    _init_dll_path()
+
+
+# When the extension module is built, we initialize it.
+# In case of an error, we do not catch the failure as it suggests there is something
+# wrong with the installation.
+_IS_TORCHAUDIO_EXT_AVAILABLE = is_module_available("torchaudio.lib._torchaudio")
+# RIR features are implemented in _torchaudio extension, but they can be individually
+# turned on/off at build time. Available means that _torchaudio is loaded properly, and
+# RIR features are found there.
+_IS_RIR_AVAILABLE = False
+_IS_ALIGN_AVAILABLE = False
+if _IS_TORCHAUDIO_EXT_AVAILABLE:
+    _load_lib("libtorchaudio")
+
+    import torchaudio.lib._torchaudio  # noqa
+
+    _check_cuda_version()
+    _IS_RIR_AVAILABLE = torchaudio.lib._torchaudio.is_rir_available()
+    _IS_ALIGN_AVAILABLE = torchaudio.lib._torchaudio.is_align_available()
+
+
+fail_if_no_rir = (
+    no_op
+    if _IS_RIR_AVAILABLE
+    else fail_with_message(
+        "requires RIR extension, but TorchAudio is not compiled with it. Please build TorchAudio with RIR support."
+    )
+)
+
+fail_if_no_align = (
+    no_op
+    if _IS_ALIGN_AVAILABLE
+    else fail_with_message(
+        "Requires alignment extension, but TorchAudio is not compiled with it. \
+        Please build TorchAudio with alignment support."
+    )
+)

source/torchaudio/_extension/utils.py

@@ -0,0 +1,133 @@
+"""Module to implement logics used for initializing extensions.
+
+The implementations here should be stateless.
+They should not depend on external state.
+Anything that depends on external state should happen in __init__.py
+"""
+import logging
+import os
+import types
+from pathlib import Path
+
+import torch
+
+_LG = logging.getLogger(__name__)
+_LIB_DIR = Path(__file__).parent.parent / "lib"
+
+
+def _get_lib_path(lib: str):
+    suffix = "pyd" if os.name == "nt" else "so"
+    path = _LIB_DIR / f"{lib}.{suffix}"
+    return path
+
+
+def _load_lib(lib: str) -> bool:
+    """Load extension module
+
+    Note:
+        In case `torchaudio` is deployed with `pex` format, the library file
+        is not in a standard location.
+        In this case, we expect that `libtorchaudio` is available somewhere
+        in the search path of dynamic loading mechanism, so that importing
+        `_torchaudio` will have library loader find and load `libtorchaudio`.
+        This is the reason why the function should not raising an error when the library
+        file is not found.
+
+    Returns:
+        bool:
+            True if the library file is found AND the library loaded without failure.
+            False if the library file is not found (like in the case where torchaudio
+            is deployed with pex format, thus the shared library file is
+            in a non-standard location.).
+            If the library file is found but there is an issue loading the library,
+            (such as missing dependency) then this function raises the exception as-is.
+
+    Raises:
+        Exception:
+            If the library file is found, but there is an issue loading the library file,
+            (when underlying `ctype.DLL` throws an exception), this function will pass
+            the exception as-is, instead of catching it and returning bool.
+            The expected case is `OSError` thrown by `ctype.DLL` when a dynamic dependency
+            is not found.
+            This behavior was chosen because the expected failure case is not recoverable.
+            If a dependency is missing, then users have to install it.
+    """
+    path = _get_lib_path(lib)
+    if not path.exists():
+        return False
+    torch.ops.load_library(path)
+    return True
+
+
+class _LazyImporter(types.ModuleType):
+    """Lazily import module/extension."""
+
+    def __init__(self, name, import_func):
+        super().__init__(name)
+        self.import_func = import_func
+        self.module = None
+
+    # Note:
+    # Python caches what was retrieved with `__getattr__`, so this method will not be
+    # called again for the same item.
+    def __getattr__(self, item):
+        self._import_once()
+        return getattr(self.module, item)
+
+    def __repr__(self):
+        if self.module is None:
+            return f"<module '{self.__module__}.{self.__class__.__name__}(\"{self.name}\")'>"
+        return repr(self.module)
+
+    def __dir__(self):
+        self._import_once()
+        return dir(self.module)
+
+    def _import_once(self):
+        if self.module is None:
+            self.module = self.import_func()
+            # Note:
+            # By attaching the module attributes to self,
+            # module attributes are directly accessible.
+            # This allows to avoid calling __getattr__ for every attribute access.
+            self.__dict__.update(self.module.__dict__)
+
+    def is_available(self):
+        try:
+            self._import_once()
+        except Exception:
+            return False
+        return True
+
+
+def _init_dll_path():
+    # On Windows Python-3.8+ has `os.add_dll_directory` call,
+    # which is called to configure dll search path.
+    # To find cuda related dlls we need to make sure the
+    # conda environment/bin path is configured Please take a look:
+    # https://stackoverflow.com/questions/59330863/cant-import-dll-module-in-python
+    # Please note: if some path can't be added using add_dll_directory we simply ignore this path
+    for path in os.environ.get("PATH", "").split(";"):
+        if os.path.exists(path):
+            try:
+                os.add_dll_directory(path)
+            except Exception:
+                pass
+
+
+def _check_cuda_version():
+    import torchaudio.lib._torchaudio
+
+    version = torchaudio.lib._torchaudio.cuda_version()
+    if version is not None and torch.version.cuda is not None:
+        version_str = str(version)
+        ta_version = f"{version_str[:-3]}.{version_str[-2]}"
+        t_version = torch.version.cuda.split(".")
+        t_version = f"{t_version[0]}.{t_version[1]}"
+        if ta_version != t_version:
+            raise RuntimeError(
+                "Detected that PyTorch and TorchAudio were compiled with different CUDA versions. "
+                f"PyTorch has CUDA version {t_version} whereas TorchAudio has CUDA version {ta_version}. "
+                "Please install the TorchAudio version that matches your PyTorch version."
+            )
+    return version

source/torchaudio/_internal/__init__.py

@@ -0,0 +1,10 @@
+try:
+    from .fb import download_url_to_file, load_state_dict_from_url
+except ImportError:
+    from torch.hub import download_url_to_file, load_state_dict_from_url
+
+
+__all__ = [
+    "load_state_dict_from_url",
+    "download_url_to_file",
+]

source/torchaudio/_internal/module_utils.py

@@ -0,0 +1,171 @@
+import importlib.util
+import os
+import warnings
+from functools import partial, wraps
+from typing import Optional
+
+
+def eval_env(var, default):
+    """Check if environment varable has True-y value"""
+    if var not in os.environ:
+        return default
+
+    val = os.environ.get(var, "0")
+    trues = ["1", "true", "TRUE", "on", "ON", "yes", "YES"]
+    falses = ["0", "false", "FALSE", "off", "OFF", "no", "NO"]
+    if val in trues:
+        return True
+    if val not in falses:
+        # fmt: off
+        raise RuntimeError(
+            f"Unexpected environment variable value `{var}={val}`. "
+            f"Expected one of {trues + falses}")
+        # fmt: on
+    return False
+
+
+def is_module_available(*modules: str) -> bool:
+    r"""Returns if a top-level module with :attr:`name` exists *without**
+    importing it. This is generally safer than try-catch block around a
+    `import X`. It avoids third party libraries breaking assumptions of some of
+    our tests, e.g., setting multiprocessing start method when imported
+    (see librosa/#747, torchvision/#544).
+    """
+    return all(importlib.util.find_spec(m) is not None for m in modules)
+
+
+def requires_module(*modules: str):
+    """Decorate function to give error message if invoked without required optional modules.
+
+    This decorator is to give better error message to users rather
+    than raising ``NameError:  name 'module' is not defined`` at random places.
+    """
+    missing = [m for m in modules if not is_module_available(m)]
+
+    if not missing:
+        # fall through. If all the modules are available, no need to decorate
+        def decorator(func):
+            return func
+
+    else:
+        req = f"module: {missing[0]}" if len(missing) == 1 else f"modules: {missing}"
+
+        def decorator(func):
+            @wraps(func)
+            def wrapped(*args, **kwargs):
+                raise RuntimeError(f"{func.__module__}.{func.__name__} requires {req}")
+
+            return wrapped
+
+    return decorator
+
+
+UNSUPPORTED = []
+
+
+def wrap_deprecated(func, name, direction: str, version: Optional[str] = None, remove: bool = False):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        message = f"{name} has been deprecated. {direction}"
+        if remove:
+            message += f' It will be removed from {"a future" if version is None else "the " + str(version)} release. '
+        warnings.warn(message, stacklevel=2)
+        return func(*args, **kwargs)
+
+    return wrapped
+
+
+def deprecated(direction: str, version: Optional[str] = None, remove: bool = False):
+    """Decorator to add deprecation message
+
+    Args:
+        direction (str): Migration steps to be given to users.
+        version (str or int): The version when the object will be removed
+        remove (bool): If enabled, append future removal message.
+    """
+
+    def decorator(func):
+        wrapped = wrap_deprecated(func, f"{func.__module__}.{func.__name__}", direction, version=version, remove=remove)
+
+        message = "This function has been deprecated. "
+        if remove:
+            message += f'It will be removed from {"future" if version is None else version} release. '
+
+        wrapped.__doc__ = f"""DEPRECATED
+
+    .. warning::
+
+       {message}
+       {direction}
+
+    {func.__doc__}
+    """
+
+        return wrapped
+
+    return decorator
+
+
+DEPRECATION_MSG = (
+    "This deprecation is part of a large refactoring effort to transition TorchAudio into a maintenance phase. "
+    "Please see https://github.com/pytorch/audio/issues/3902 for more information."
+)
+
+IO_DEPRECATION_MSG = (
+    "This deprecation is part of a large refactoring effort to transition TorchAudio into a maintenance phase. "
+    "The decoding and encoding capabilities of PyTorch for both audio"
+    " and video are being consolidated into TorchCodec. "
+    "Please see https://github.com/pytorch/audio/issues/3902 for more information."
+)
+
+dropping_support = deprecated(DEPRECATION_MSG, version="2.9", remove=True)
+
+
+def dropping_class_support(c, msg=DEPRECATION_MSG):
+    c.__init__ = wrap_deprecated(c.__init__, f"{c.__module__}.{c.__name__}", msg, version="2.9", remove=True)
+    c.__doc__ = f"""DEPRECATED
+
+.. warning::
+
+    This class is deprecated from version 2.8. It will be removed in the 2.9 release.
+    {msg}
+{c.__doc__}
+"""
+
+    UNSUPPORTED.append(c)
+    return c
+
+
+def dropping_const_support(c, msg=DEPRECATION_MSG, name=None):
+    c.__doc__ = f"""[DEPRECATED]
+
+.. warning::
+
+    This object is deprecated deprecated from version 2.8. It will be removed in the 2.9 release.
+    {msg}
+{c.__doc__}
+    """
+    return c
+
+
+dropping_class_io_support = partial(dropping_class_support, msg=IO_DEPRECATION_MSG)
+
+dropping_io_support = deprecated(IO_DEPRECATION_MSG, version="2.9", remove=True)
+
+
+def fail_with_message(message):
+    """Generate decorator to give users message about missing TorchAudio extension."""
+
+    def decorator(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            raise RuntimeError(f"{func.__module__}.{func.__name__} {message}")
+
+        return wrapped
+
+    return decorator
+
+
+def no_op(func):
+    """Op-op decorator. Used in place of fail_with_message when a functionality that requires extension works fine."""
+    return func

source/torchaudio/_torchcodec.py

@@ -0,0 +1,340 @@
+"""TorchCodec integration for TorchAudio."""
+
+import os
+from typing import BinaryIO, Optional, Tuple, Union
+
+import torch
+
+
+def load_with_torchcodec(
+    uri: Union[BinaryIO, str, os.PathLike],
+    frame_offset: int = 0,
+    num_frames: int = -1,
+    normalize: bool = True,
+    channels_first: bool = True,
+    format: Optional[str] = None,
+    buffer_size: int = 4096,
+    backend: Optional[str] = None,
+) -> Tuple[torch.Tensor, int]:
+    """Load audio data from source using TorchCodec's AudioDecoder.
+
+    .. note::
+
+        This function supports the same API as :func:`~torchaudio.load`, and
+        relies on TorchCodec's decoding capabilities under the hood. It is
+        provided for convenience, but we do recommend that you port your code to
+        natively use ``torchcodec``'s ``AudioDecoder`` class for better
+        performance:
+        https://docs.pytorch.org/torchcodec/stable/generated/torchcodec.decoders.AudioDecoder.
+        As of TorchAudio 2.9, :func:`~torchaudio.load` relies on
+        :func:`~torchaudio.load_with_torchcodec`. Note that some parameters of
+        :func:`~torchaudio.load`, like ``normalize``, ``buffer_size``, and
+        ``backend``, are ignored by :func:`~torchaudio.load_with_torchcodec`.
+        To install torchcodec, follow the instructions at https://github.com/pytorch/torchcodec#installing-torchcodec.
+
+
+    Args:
+        uri (path-like object or file-like object):
+            Source of audio data. The following types are accepted:
+
+            * ``path-like``: File path or URL.
+            * ``file-like``: Object with ``read(size: int) -> bytes`` method.
+
+        frame_offset (int, optional):
+            Number of samples to skip before start reading data.
+        num_frames (int, optional):
+            Maximum number of samples to read. ``-1`` reads all the remaining samples,
+            starting from ``frame_offset``.
+        normalize (bool, optional):
+            TorchCodec always returns normalized float32 samples. This parameter
+            is ignored and a warning is issued if set to False.
+            Default: ``True``.
+        channels_first (bool, optional):
+            When True, the returned Tensor has dimension `[channel, time]`.
+            Otherwise, the returned Tensor's dimension is `[time, channel]`.
+        format (str or None, optional):
+            Format hint for the decoder. May not be supported by all TorchCodec
+            decoders. (Default: ``None``)
+        buffer_size (int, optional):
+            Not used by TorchCodec AudioDecoder. Provided for API compatibility.
+        backend (str or None, optional):
+            Not used by TorchCodec AudioDecoder. Provided for API compatibility.
+
+    Returns:
+        (torch.Tensor, int): Resulting Tensor and sample rate.
+        Always returns float32 tensors. If ``channels_first=True``, shape is
+        `[channel, time]`, otherwise `[time, channel]`.
+
+    Raises:
+        ImportError: If torchcodec is not available.
+        ValueError: If unsupported parameters are used.
+        RuntimeError: If TorchCodec fails to decode the audio.
+
+    Note:
+        - TorchCodec always returns normalized float32 samples, so the ``normalize``
+          parameter has no effect.
+        - The ``buffer_size`` and ``backend`` parameters are ignored.
+        - Not all audio formats supported by torchaudio backends may be supported
+          by TorchCodec.
+    """
+    # Import torchcodec here to provide clear error if not available
+    try:
+        from torchcodec.decoders import AudioDecoder
+    except ImportError as e:
+        raise ImportError(
+            "TorchCodec is required for load_with_torchcodec. " "Please install torchcodec to use this function."
+        ) from e
+
+    # Parameter validation and warnings
+    if not normalize:
+        import warnings
+
+        warnings.warn(
+            "TorchCodec AudioDecoder always returns normalized float32 samples. "
+            "The 'normalize=False' parameter is ignored.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    if buffer_size != 4096:
+        import warnings
+
+        warnings.warn("The 'buffer_size' parameter is not used by TorchCodec AudioDecoder.", UserWarning, stacklevel=2)
+
+    if backend is not None:
+        import warnings
+
+        warnings.warn("The 'backend' parameter is not used by TorchCodec AudioDecoder.", UserWarning, stacklevel=2)
+
+    if format is not None:
+        import warnings
+
+        warnings.warn("The 'format' parameter is not supported by TorchCodec AudioDecoder.", UserWarning, stacklevel=2)
+
+    # Create AudioDecoder
+    try:
+        decoder = AudioDecoder(uri)
+    except Exception as e:
+        raise RuntimeError(f"Failed to create AudioDecoder for {uri}: {e}") from e
+
+    # Get sample rate from metadata
+    sample_rate = decoder.metadata.sample_rate
+    if sample_rate is None:
+        raise RuntimeError("Unable to determine sample rate from audio metadata")
+
+    # Decode the entire file first, then subsample manually
+    # This is the simplest approach since torchcodec uses time-based indexing
+    try:
+        audio_samples = decoder.get_all_samples()
+    except Exception as e:
+        raise RuntimeError(f"Failed to decode audio samples: {e}") from e
+
+    data = audio_samples.data
+
+    # Apply frame_offset and num_frames (which are actually sample offsets)
+    if frame_offset > 0:
+        if frame_offset >= data.shape[1]:
+            # Return empty tensor if offset is beyond available data
+            empty_shape = (data.shape[0], 0) if channels_first else (0, data.shape[0])
+            return torch.zeros(empty_shape, dtype=torch.float32), sample_rate
+        data = data[:, frame_offset:]
+
+    if num_frames == 0:
+        # Return empty tensor if num_frames is 0
+        empty_shape = (data.shape[0], 0) if channels_first else (0, data.shape[0])
+        return torch.zeros(empty_shape, dtype=torch.float32), sample_rate
+    elif num_frames > 0:
+        data = data[:, :num_frames]
+
+    # TorchCodec returns data in [channel, time] format by default
+    # Handle channels_first parameter
+    if not channels_first:
+        data = data.transpose(0, 1)  # [channel, time] -> [time, channel]
+
+    return data, sample_rate
+
+
+def save_with_torchcodec(
+    uri: Union[str, os.PathLike],
+    src: torch.Tensor,
+    sample_rate: int,
+    channels_first: bool = True,
+    format: Optional[str] = None,
+    encoding: Optional[str] = None,
+    bits_per_sample: Optional[int] = None,
+    buffer_size: int = 4096,
+    backend: Optional[str] = None,
+    compression: Optional[Union[float, int]] = None,
+) -> None:
+    """Save audio data to file using TorchCodec's AudioEncoder.
+
+    .. note::
+
+        This function supports the same API as :func:`~torchaudio.save`, and
+        relies on TorchCodec's encoding capabilities under the hood. It is
+        provided for convenience, but we do recommend that you port your code to
+        natively use ``torchcodec``'s ``AudioEncoder`` class for better
+        performance:
+        https://docs.pytorch.org/torchcodec/stable/generated/torchcodec.encoders.AudioEncoder.
+        As of TorchAudio 2.9, :func:`~torchaudio.save` relies on
+        :func:`~torchaudio.save_with_torchcodec`. Note that some parameters of
+        :func:`~torchaudio.save`, like ``format``, ``encoding``,
+        ``bits_per_sample``, ``buffer_size``, and ``backend``, are ignored by
+        are ignored by :func:`~torchaudio.save_with_torchcodec`.
+        To install torchcodec, follow the instructions at https://github.com/pytorch/torchcodec#installing-torchcodec.
+
+    This function provides a TorchCodec-based alternative to torchaudio.save
+    with the same API. TorchCodec's AudioEncoder provides efficient encoding
+    with FFmpeg under the hood.
+
+    Args:
+        uri (path-like object):
+            Path to save the audio file. The file extension determines the format.
+
+        src (torch.Tensor):
+            Audio data to save. Must be a 1D or 2D tensor with float32 values
+            in the range [-1, 1]. If 2D, shape should be [channel, time] when
+            channels_first=True, or [time, channel] when channels_first=False.
+
+        sample_rate (int):
+            Sample rate of the audio data.
+
+        channels_first (bool, optional):
+            Indicates whether the input tensor has channels as the first dimension.
+            If True, expects [channel, time]. If False, expects [time, channel].
+            Default: True.
+
+        format (str or None, optional):
+            Audio format hint. Not used by TorchCodec (format is determined by
+            file extension). A warning is issued if provided.
+            Default: None.
+
+        encoding (str or None, optional):
+            Audio encoding. Not fully supported by TorchCodec AudioEncoder.
+            A warning is issued if provided. Default: None.
+
+        bits_per_sample (int or None, optional):
+            Bits per sample. Not directly supported by TorchCodec AudioEncoder.
+            A warning is issued if provided. Default: None.
+
+        buffer_size (int, optional):
+            Not used by TorchCodec AudioEncoder. Provided for API compatibility.
+            A warning is issued if not default value. Default: 4096.
+
+        backend (str or None, optional):
+            Not used by TorchCodec AudioEncoder. Provided for API compatibility.
+            A warning is issued if provided. Default: None.
+
+        compression (float, int or None, optional):
+            Compression level or bit rate. Maps to bit_rate parameter in
+            TorchCodec AudioEncoder. Default: None.
+
+    Raises:
+        ImportError: If torchcodec is not available.
+        ValueError: If input parameters are invalid.
+        RuntimeError: If TorchCodec fails to encode the audio.
+
+    Note:
+        - TorchCodec AudioEncoder expects float32 samples in [-1, 1] range.
+        - Some parameters (format, encoding, bits_per_sample, buffer_size, backend)
+          are not used by TorchCodec but are provided for API compatibility.
+        - The output format is determined by the file extension in the uri.
+        - TorchCodec uses FFmpeg under the hood for encoding.
+    """
+    # Import torchcodec here to provide clear error if not available
+    try:
+        from torchcodec.encoders import AudioEncoder
+    except ImportError as e:
+        raise ImportError(
+            "TorchCodec is required for save_with_torchcodec. " "Please install torchcodec to use this function."
+        ) from e
+
+    # Parameter validation and warnings
+    if format is not None:
+        import warnings
+
+        warnings.warn(
+            "The 'format' parameter is not used by TorchCodec AudioEncoder. "
+            "Format is determined by the file extension.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    if encoding is not None:
+        import warnings
+
+        warnings.warn(
+            "The 'encoding' parameter is not fully supported by TorchCodec AudioEncoder.", UserWarning, stacklevel=2
+        )
+
+    if bits_per_sample is not None:
+        import warnings
+
+        warnings.warn(
+            "The 'bits_per_sample' parameter is not directly supported by TorchCodec AudioEncoder.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    if buffer_size != 4096:
+        import warnings
+
+        warnings.warn("The 'buffer_size' parameter is not used by TorchCodec AudioEncoder.", UserWarning, stacklevel=2)
+
+    if backend is not None:
+        import warnings
+
+        warnings.warn("The 'backend' parameter is not used by TorchCodec AudioEncoder.", UserWarning, stacklevel=2)
+
+    # Input validation
+    if not isinstance(src, torch.Tensor):
+        raise ValueError(f"Expected src to be a torch.Tensor, got {type(src)}")
+
+    if src.dtype != torch.float32:
+        src = src.float()
+
+    if sample_rate <= 0:
+        raise ValueError(f"sample_rate must be positive, got {sample_rate}")
+
+    # Handle tensor shape and channels_first
+    if src.ndim == 1:
+        # Convert to 2D: [1, time] for channels_first=True
+        if channels_first:
+            data = src.unsqueeze(0)  # [1, time]
+        else:
+            # For channels_first=False, input is [time] -> reshape to [time, 1] -> transpose to [1, time]
+            data = src.unsqueeze(1).transpose(0, 1)  # [time, 1] -> [1, time]
+    elif src.ndim == 2:
+        if channels_first:
+            data = src  # Already [channel, time]
+        else:
+            data = src.transpose(0, 1)  # [time, channel] -> [channel, time]
+    else:
+        raise ValueError(f"Expected 1D or 2D tensor, got {src.ndim}D tensor")
+
+    # Create AudioEncoder
+    try:
+        encoder = AudioEncoder(data, sample_rate=sample_rate)
+    except Exception as e:
+        raise RuntimeError(f"Failed to create AudioEncoder: {e}") from e
+
+    # Determine bit_rate from compression parameter
+    bit_rate = None
+    if compression is not None:
+        if isinstance(compression, (int, float)):
+            bit_rate = int(compression)
+        else:
+            import warnings
+
+            warnings.warn(
+                f"Unsupported compression type {type(compression)}. "
+                "TorchCodec AudioEncoder expects int or float for bit_rate.",
+                UserWarning,
+                stacklevel=2,
+            )
+
+    # Save to file
+    try:
+        encoder.to_file(uri, bit_rate=bit_rate)
+    except Exception as e:
+        raise RuntimeError(f"Failed to save audio to {uri}: {e}") from e