| pub const VERSION: &str = env!("CARGO_PKG_VERSION"); | |
| mod arc_rwlock_serde; | |
| pub mod decoders; | |
| pub mod encoding; | |
| pub mod models; | |
| pub mod normalizers; | |
| pub mod pre_tokenizers; | |
| pub mod processors; | |
| pub mod tasks; | |
| pub mod tokenizer; | |
| pub mod trainers; | |
| pub mod utils; | |