| [package] | |
| name = "tokenizers-python" | |
| version = "0.20.1" | |
| authors = ["Anthony MOI <m.anthony.moi@gmail.com>"] | |
| edition = "2021" | |
| [lib] | |
| name = "tokenizers" | |
| crate-type = ["cdylib"] | |
| [dependencies] | |
| rayon = "1.10" | |
| serde = { version = "1.0", features = [ "rc", "derive" ]} | |
| serde_json = "1.0" | |
| libc = "0.2" | |
| env_logger = "0.11" | |
| pyo3 = { version = "0.21" } | |
| numpy = "0.21" | |
| ndarray = "0.15" | |
| itertools = "0.12" | |
| [dependencies.tokenizers] | |
| path = "../../tokenizers" | |
| [dev-dependencies] | |
| tempfile = "3.10" | |
| pyo3 = { version = "0.21", features = ["auto-initialize"] } | |
| [features] | |
| defaut = ["pyo3/extension-module"] | |