#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/stl_bind.h>
#include "tokenizer.hpp"

namespace py = pybind11;

PYBIND11_MODULE(_cpp_tokenizer, m) {
    m.doc() = "Fast C++ BPE tokenizer for curriculum-trained vocabularies";

    py::class_<curriculum::Tokenizer>(m, "Tokenizer")
        .def(py::init<>())
        .def("load", &curriculum::Tokenizer::load,
             py::arg("vocab_path"), py::arg("merges_path"),
             "Load tokenizer from vocab.json and merges.txt files")
        .def("encode", &curriculum::Tokenizer::encode,
             py::arg("text"),
             py::call_guard<py::gil_scoped_release>(),
             "Encode text to list of token IDs")
        .def("decode", &curriculum::Tokenizer::decode,
             py::arg("ids"),
             py::call_guard<py::gil_scoped_release>(),
             "Decode list of token IDs to text")
        .def("encode_batch", &curriculum::Tokenizer::encode_batch,
             py::arg("texts"), py::arg("num_threads") = 0,
             py::call_guard<py::gil_scoped_release>(),
             "Batch encode multiple texts")
        .def("set_cache_capacity", &curriculum::Tokenizer::set_cache_capacity,
             py::arg("capacity"), "Set the bounded piece-cache capacity; zero disables caching")
        .def("clear_cache", &curriculum::Tokenizer::clear_cache,
             "Clear cached pieces and cache counters")
        .def("cache_info", &curriculum::Tokenizer::cache_info,
             "Get cache capacity, entries, hits, misses, and payload bytes")
        .def("vocab_size", &curriculum::Tokenizer::vocab_size,
             "Get vocabulary size")
        .def("id_to_token", &curriculum::Tokenizer::id_to_token,
             py::arg("id"),
             "Get token string for an ID")
        .def("token_to_id", &curriculum::Tokenizer::token_to_id,
             py::arg("token"),
             "Get ID for a token string (-1 if not found)")
        .def("set_special_tokens", &curriculum::Tokenizer::set_special_tokens,
             py::arg("tokens"),
             "Register existing vocabulary tokens as atomic controls")
        .def("set_special_token_aliases", &curriculum::Tokenizer::set_special_token_aliases,
             py::arg("aliases"),
             "Register explicit legacy decode-only aliases without changing learned tokens")
        .def("configure_special_tokens", &curriculum::Tokenizer::configure_special_tokens,
             py::arg("token_ids"), py::arg("aliases"), py::arg("expected_vocab_size"),
             "Validate configured controls and append only explicitly declared legacy tail IDs");
}