File size: 917 Bytes
72c0672 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | use pyo3::prelude::*;
use tk::Token;
#[pyclass(module = "tokenizers", name = "Token")]
#[derive(Clone)]
pub struct PyToken {
token: Token,
}
impl From<Token> for PyToken {
fn from(token: Token) -> Self {
Self { token }
}
}
impl From<PyToken> for Token {
fn from(token: PyToken) -> Self {
token.token
}
}
#[pymethods]
impl PyToken {
#[new]
#[pyo3(text_signature = None)]
fn new(id: u32, value: String, offsets: (usize, usize)) -> PyToken {
Token::new(id, value, offsets).into()
}
#[getter]
fn get_id(&self) -> u32 {
self.token.id
}
#[getter]
fn get_value(&self) -> &str {
&self.token.value
}
#[getter]
fn get_offsets(&self) -> (usize, usize) {
self.token.offsets
}
fn as_tuple(&self) -> (u32, &str, (usize, usize)) {
(self.token.id, &self.token.value, self.token.offsets)
}
}
|