superbpe/tokenizers_superbpe/bindings/node/src/lib.rs

#![deny(clippy::all)]

pub const VERSION: &str = env!("CARGO_PKG_VERSION");

mod arc_rwlock_serde;
pub mod decoders;
pub mod encoding;
pub mod models;
pub mod normalizers;
pub mod pre_tokenizers;
pub mod processors;
pub mod tasks;
pub mod tokenizer;
pub mod trainers;
pub mod utils;