Fix public tokenizer batch shape before 1.0.2 release

CHANGELOG.md

@@ -10,6 +10,8 @@ Accuracy is 841-dev, union at 0.95, 354 gold spans. A version that changes those
 
 ## 1.0.2
 
+Fix the public wrapper’s tokenizer batch shape so local model inference works. The packed tokenizer regression and an end-to-end local inference check cover this path.
+
 Fix immediately labelled hyphenated country-area phones with one-digit country codes or wider area codes. Numeric continuations, slash lists, weak contact cues, and nearby title prose still abstain. Same weights, threshold, and API.
 
 841-dev: rules recover 14 more whole spans; the union recovers one (323 → 324), reducing residual passages 25 → 24. Rules FP remain 98; union FP remain 123. Lost gold, new false characters, and new clean-passage damage are all zero. Rules SHA `3016ae5b…`.

nergal.py

@@ -142,7 +142,7 @@ class Encoding:
         self.pieces = lru_cache(maxsize=16384)(lambda s: tuple(tokenizer.encode(s, add_special_tokens=False)))
 
     def encode(self, words):
-        encoded = self.tokenizer(words, is_split_into_words=True, truncation=False, padding=False)
+        encoded = self.tokenizer([words], is_split_into_words=True, truncation=False, padding=False)
         ids = encoded['input_ids'][0]
         mapping = encoded.word_ids(0)
         first, actual = {}, {}

test_nergal.py

@@ -24,6 +24,17 @@ class NergalTests(unittest.TestCase):
         digest = hashlib.sha256((HERE / 'scrub_pii.py').read_bytes()).hexdigest()
         self.assertEqual(digest, RULES_SHA)
 
+    def test_real_tokenizer_preserves_batch_and_unit_alignment(self):
+        from transformers import AutoTokenizer
+        from nergal import Encoding
+        tokenizer = AutoTokenizer.from_pretrained(str(HERE), local_files_only=True, fix_mistral_regex=False)
+        encoding = Encoding(tokenizer)
+        words = ['A', '[PII_SPACE]', '1']
+        encoded, first = encoding.encode(words)
+        self.assertIsInstance(encoded['input_ids'][0], list)
+        self.assertEqual(len(first), len(words))
+        self.assertEqual([encoded.word_ids(0)[i] for i in first], [0, 1, 2])
+
     def test_union_keeps_regex_and_adds_model_spans(self):
         from nergal import apply_union, scrub_spans
         text = 'Ring 000000000 then extra.'