Update tokenizer.py

tokenizer.py

@@ -1,9 +1,10 @@
-"""Tokenizer - MODIFIED for conversations"""
+"""Tokenizer for Veda Programming Assistant"""
 
 import json
 import re
 from typing import List, Dict, Optional
 
+
 class VedaTokenizer:
     """Tokenizer with conversation support"""
     
@@ -15,18 +16,16 @@ class VedaTokenizer:
     
     def _init_vocab(self):
         """Initialize vocabulary with conversation tokens"""
-        # Special tokens - ADDED conversation tokens
         special = [
             "<PAD>", "<UNK>", "<START>", "<END>",
-            "<CODE>", "<ENDCODE>",  # For code blocks
-            "<USER>", "<ASSISTANT>"  # For conversation
+            "<CODE>", "<ENDCODE>",
+            "<USER>", "<ASSISTANT>"
         ]
         
         for idx, token in enumerate(special):
             self.token_to_idx[token] = idx
             self.idx_to_token[idx] = token
         
-        # ASCII characters
         idx = len(special)
         for i in range(32, 127):
             char = chr(i)
@@ -34,7 +33,6 @@ class VedaTokenizer:
             self.idx_to_token[idx] = char
             idx += 1
         
-        # Whitespace
         for char in ["\n", "\t"]:
             self.token_to_idx[char] = idx
             self.idx_to_token[idx] = char
@@ -115,23 +113,21 @@ class VedaTokenizer:
         return tokens
     
     def decode(self, indices: List[int]) -> str:
-        """Decode indices to text - MODIFIED for conversation tokens"""
+        """Decode indices to text"""
         result = []
         prev = ""
         
         for idx in indices:
-            if idx == 0:  # PAD
+            if idx == 0:
                 continue
             if idx not in self.idx_to_token:
                 continue
             
             token = self.idx_to_token[idx]
             
-            # Skip special tokens in output
             if token in ["<PAD>", "<UNK>", "<START>", "<END>", "<USER>", "<ASSISTANT>"]:
                 continue
             
-            # Handle code blocks
             if token == "<CODE>":
                 result.append("\n```python\n")
                 prev = "\n"
@@ -141,7 +137,6 @@ class VedaTokenizer:
                 prev = "\n"
                 continue
             
-            # Smart joining
             if not result:
                 result.append(token)
             elif token in "\n\t":
@@ -150,7 +145,7 @@ class VedaTokenizer:
                 result.append(token)
             elif prev in "(\n\t[{":
                 result.append(token)
-            elif prev.isalnum() and len(token) > 0 and token[0].isalnum():
+            elif len(prev) > 0 and prev[-1].isalnum() and len(token) > 0 and token[0].isalnum():
                 result.append(" " + token)
             else:
                 result.append(token)
@@ -160,6 +155,7 @@ class VedaTokenizer:
         return "".join(result)
     
     def save(self, path: str):
+        """Save tokenizer"""
         with open(path, 'w') as f:
             json.dump({
                 'vocab_size': self.vocab_size,
@@ -169,6 +165,7 @@ class VedaTokenizer:
             }, f, indent=2)
     
     def load(self, path: str):
+        """Load tokenizer"""
         with open(path, 'r') as f:
             data = json.load(f)
         self.vocab_size = data['vocab_size']