File size: 2,622 Bytes
c941ae3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/stl_bind.h>
#include "tokenizer.hpp"
namespace py = pybind11;
PYBIND11_MODULE(_cpp_tokenizer, m) {
m.doc() = "Fast C++ BPE tokenizer for curriculum-trained vocabularies";
py::class_<curriculum::Tokenizer>(m, "Tokenizer")
.def(py::init<>())
.def("load", &curriculum::Tokenizer::load,
py::arg("vocab_path"), py::arg("merges_path"),
"Load tokenizer from vocab.json and merges.txt files")
.def("encode", &curriculum::Tokenizer::encode,
py::arg("text"),
py::call_guard<py::gil_scoped_release>(),
"Encode text to list of token IDs")
.def("decode", &curriculum::Tokenizer::decode,
py::arg("ids"),
py::call_guard<py::gil_scoped_release>(),
"Decode list of token IDs to text")
.def("encode_batch", &curriculum::Tokenizer::encode_batch,
py::arg("texts"), py::arg("num_threads") = 0,
py::call_guard<py::gil_scoped_release>(),
"Batch encode multiple texts")
.def("set_cache_capacity", &curriculum::Tokenizer::set_cache_capacity,
py::arg("capacity"), "Set the bounded piece-cache capacity; zero disables caching")
.def("clear_cache", &curriculum::Tokenizer::clear_cache,
"Clear cached pieces and cache counters")
.def("cache_info", &curriculum::Tokenizer::cache_info,
"Get cache capacity, entries, hits, misses, and payload bytes")
.def("vocab_size", &curriculum::Tokenizer::vocab_size,
"Get vocabulary size")
.def("id_to_token", &curriculum::Tokenizer::id_to_token,
py::arg("id"),
"Get token string for an ID")
.def("token_to_id", &curriculum::Tokenizer::token_to_id,
py::arg("token"),
"Get ID for a token string (-1 if not found)")
.def("set_special_tokens", &curriculum::Tokenizer::set_special_tokens,
py::arg("tokens"),
"Register existing vocabulary tokens as atomic controls")
.def("set_special_token_aliases", &curriculum::Tokenizer::set_special_token_aliases,
py::arg("aliases"),
"Register explicit legacy decode-only aliases without changing learned tokens")
.def("configure_special_tokens", &curriculum::Tokenizer::configure_special_tokens,
py::arg("token_ids"), py::arg("aliases"), py::arg("expected_vocab_size"),
"Validate configured controls and append only explicitly declared legacy tail IDs");
}
|