Update tokenizer.py

tokenizer.py

@@ -122,7 +122,7 @@ class ParadigmFinderSegmenter:
     def segment_with_alignment(self, raw_text: str) -> Tuple[str, List[Optional[int]]]:
         """
         Preprocess + segment; return segmented text and a char map from segmented
-        text back to raw indices (None for inserted spaces).
+        text back to raw indices.
         """
         # 1) Preprocess with alignment
         pre_chars, pre_map = [], []
@@ -260,7 +260,7 @@ class ParadigmTokenizerWrapper(PreTrainedTokenizerFast):
     slow_tokenizer_class = None
     
     def __init__(self, *args, **kwargs):
-        # ensure fast tokenizer is loaded directly (no slow->fast conversion)
+
         name_or_path = kwargs.get("name_or_path", None)
         if name_or_path is None and len(args) > 0 and isinstance(args[0], str):
             name_or_path = args[0]
@@ -273,10 +273,6 @@ class ParadigmTokenizerWrapper(PreTrainedTokenizerFast):
 
         super().__init__(*args, **kwargs)
 
-        # The folder path AutoTokenizer passes becomes available as:
-        #  - kwargs.get("name_or_path") on first init
-        #  - or self.name_or_path after init
-        # new:
         repo_id_or_path = kwargs.get("name_or_path", getattr(self, "name_or_path", None)) \
                   or os.path.dirname(getattr(self, "tokenizer_file", "")) or "."
         revision = kwargs.get("revision", None)
@@ -296,30 +292,44 @@ class ParadigmTokenizerWrapper(PreTrainedTokenizerFast):
             space_punct=cfg.get("space_punct", True),
         )
 
-    # ---- main entry point ----
     def __call__(self, text, **kwargs):
-            if isinstance(text, str):
-                seg, _ = self.segmenter.segment_with_alignment(text)
-                return super().__call__(seg, **kwargs)
-            try:
-                items = list(text)
-            except TypeError:
-                # single non-str item (e.g., tuple)
-                s = _coerce_to_str(text)
-                seg, _ = self.segmenter.segment_with_alignment(s)
-                return super().__call__(seg, **kwargs)
-        
-            segs = []
-            for t in items:
-                s = _coerce_to_str(t)
-                seg, _ = self.segmenter.segment_with_alignment(s)
-                segs.append(seg)
-            return super().__call__(segs, **kwargs)
-
-    def tokenize(self, text, **kwargs):
+        # 1) fast path: already a plain string
         if isinstance(text, str):
             seg, _ = self.segmenter.segment_with_alignment(text)
+            return super().__call__(seg, **kwargs)
+    
+        # 2) dicts: coerce to a single string (don't iterate keys!)
+        if isinstance(text, dict):
+            s = _coerce_to_str(text)
+            seg, _ = self.segmenter.segment_with_alignment(s)
+            return super().__call__(seg, **kwargs)
+    
+        # 3) sequences (list/tuple/etc.): coerce each element to a string
+        try:
+            items = list(text)
+        except TypeError:
+            s = _coerce_to_str(text)
+            seg, _ = self.segmenter.segment_with_alignment(s)
+            return super().__call__(seg, **kwargs)
+    
+        segs = []
+        for t in items:
+            s = _coerce_to_str(t)
+            seg, _ = self.segmenter.segment_with_alignment(s)
+            segs.append(seg)
+        return super().__call__(segs, **kwargs)
+    
+    
+    def tokenize(self, text, **kwargs):
+        if isinstance(text, str):
+            seg, _ = self.segmenter.segment_with_alignment(text)   # <-- fix here
+            return super().tokenize(seg, **kwargs)
+    
+        if isinstance(text, dict):
+            s = _coerce_to_str(text)
+            seg, _ = self.segmenter.segment_with_alignment(s)
             return super().tokenize(seg, **kwargs)
+    
         try:
             items = list(text)
         except TypeError: