Upload handler.py

handler.py

@@ -135,6 +135,27 @@ class _SimpleTokenizer:
         tokens.append(self.eos_token_id)
         return tokens
 
+    def decode(self, token_ids: Sequence[int]) -> str:
+        """Convert token IDs back into a text string."""
+
+        characters: list[str] = []
+        for idx in token_ids:
+            if idx == self.eos_token_id:
+                break
+            if idx in {self.pad_token_id, self.bos_token_id}:
+                continue
+
+            if 0 <= idx < len(self._tokens):
+                token = self._tokens[idx]
+                if token not in {"<pad>", "<bos>", "<eos>", "<unk>"}:
+                    characters.append(token)
+                else:
+                    characters.append("?")
+            else:
+                characters.append("?")
+
+        return "".join(characters)
+
 
 def _summarise_intent(psi: Sequence[float], top_k: int = 4) -> list[str]:
     """Convert strongest symbolic dimensions into descriptors."""