Spaces:
Runtime error
Runtime error
File size: 3,959 Bytes
31e8a9f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Torch-free tokenizer (PRD §4.4).
Drop-in replacement for ``transformers.AutoTokenizer`` in production. The
HuggingFace ``tokenizers`` library is a small Rust extension that reads the same
``tokenizer.json`` a fine-tuned model ships with, but never imports torch.
Differences from the PRD sketch (see DECISIONS.md):
* Loads from a local ``tokenizer.json`` file (downloaded with the ONNX model)
instead of ``Tokenizer.from_pretrained`` — no network at inference time, and
the base ``microsoft/deberta-v3-base`` repo does not ship a ``tokenizer.json``.
* Pads to the longest sequence in the batch instead of always to 512 tokens.
The ONNX graphs have dynamic axes, so this is numerically identical and
several times faster on a shared CPU.
* The pad id/token are read from the vocabulary instead of hardcoded to 0.
* Pair inputs use ``only_first`` truncation so the (short) aspect sentence is
never cut — only the review body is truncated.
"""
from __future__ import annotations
from pathlib import Path
import numpy as np
from tokenizers import Encoding, Tokenizer
_PAD_CANDIDATES = ("[PAD]", "<pad>", "<PAD>")
DEFAULT_MAX_LENGTH = 512
class LightTokenizer:
"""Rust-backed tokenizer producing int64 numpy batches for onnxruntime."""
def __init__(self, tokenizer_path: str | Path, max_length: int | None = None) -> None:
"""
Args:
tokenizer_path: ``tokenizer.json`` exported with the model.
max_length: Truncation length. ``None`` uses the length stored in the
file's truncation settings (written at export so serving matches
training), falling back to 512.
"""
path = Path(tokenizer_path)
if not path.is_file():
raise FileNotFoundError(f"tokenizer.json not found: {path}")
self._tok = Tokenizer.from_file(str(path))
stored = (self._tok.truncation or {}).get("max_length")
self.max_length = int(max_length or stored or DEFAULT_MAX_LENGTH)
pad_token = next((t for t in _PAD_CANDIDATES if self._tok.token_to_id(t) is not None), None)
if pad_token is None:
raise ValueError(f"No padding token {_PAD_CANDIDATES} in vocabulary of {path}")
self.pad_token = pad_token
self.pad_id = int(self._tok.token_to_id(pad_token))
self._tok.enable_padding(pad_id=self.pad_id, pad_token=self.pad_token)
# ------------------------------------------------------------------
def __call__(self, text: str, text_pair: str | None = None) -> dict[str, np.ndarray]:
"""Encode one text (or one text pair) into a batch of size 1."""
if text_pair is None:
return self.encode_batch([text])
return self.encode_pairs([(text, text_pair)])
def encode_batch(self, texts: list[str]) -> dict[str, np.ndarray]:
"""Encode single sequences, padded to the longest in the batch."""
if not texts:
raise ValueError("encode_batch() needs at least one text")
self._tok.enable_truncation(max_length=self.max_length, strategy="longest_first")
return self._to_arrays(self._tok.encode_batch(texts))
def encode_pairs(self, pairs: list[tuple[str, str]]) -> dict[str, np.ndarray]:
"""Encode (first, second) pairs; only the first sequence is ever truncated."""
if not pairs:
raise ValueError("encode_pairs() needs at least one pair")
self._tok.enable_truncation(max_length=self.max_length, strategy="only_first")
return self._to_arrays(self._tok.encode_batch(pairs))
@staticmethod
def _to_arrays(encodings: list[Encoding]) -> dict[str, np.ndarray]:
return {
"input_ids": np.asarray([e.ids for e in encodings], dtype=np.int64),
"attention_mask": np.asarray([e.attention_mask for e in encodings], dtype=np.int64),
"token_type_ids": np.asarray([e.type_ids for e in encodings], dtype=np.int64),
}
|