Upload 4 files

tevunahai_quant_info.json

@@ -0,0 +1,8 @@
+{
+  "quant_method": "fp8",
+  "fp8_dtype": "e4m3",
+  "per_channel": true,
+  "calibration_samples": 2048,
+  "quantized_by": "TevunahAi",
+  "timestamp": "2025-12-05T01:53:52.287326"
+}

tokenization_grok2.py

@@ -0,0 +1,164 @@
+"""
+Grok-2 Tokenizer
+
+Custom tokenizer for xAI's Grok-2 model using the .tok.json format.
+"""
+
+import json
+from typing import List, Optional, Union
+from transformers import PreTrainedTokenizer
+
+
+class Grok2Tokenizer(PreTrainedTokenizer):
+    """
+    Tokenizer for Grok-2 model.
+    
+    This tokenizer uses a byte-level BPE vocabulary stored in tokenizer.tok.json format.
+    """
+    
+    vocab_files_names = {"vocab_file": "tokenizer.tok.json"}
+    model_input_names = ["input_ids", "attention_mask"]
+    
+    def __init__(
+        self,
+        vocab_file: str,
+        bos_token: str = "<|begin_of_text|>",
+        eos_token: str = "<|end_of_text|>",
+        pad_token: str = "<|pad|>",
+        unk_token: str = "<|unk|>",
+        **kwargs,
+    ):
+        # Load the vocabulary
+        with open(vocab_file, "r", encoding="utf-8") as f:
+            vocab_data = json.load(f)
+        
+        self.reserved_tokens = vocab_data.get("reserved_tokens", 128)
+        self.regular_tokens = vocab_data.get("regular_tokens", [])
+        
+        # Build byte to token mapping
+        self._byte_to_token = {}
+        self._token_to_bytes = {}
+        
+        for entry in self.regular_tokens:
+            token_id = entry["token"]
+            byte_seq = bytes(entry["bytes"])
+            self._byte_to_token[byte_seq] = token_id
+            self._token_to_bytes[token_id] = byte_seq
+        
+        # Build vocabulary dict for compatibility
+        self._vocab = {}
+        for entry in self.regular_tokens:
+            token_id = entry["token"]
+            # Create a string representation
+            byte_seq = bytes(entry["bytes"])
+            try:
+                text = byte_seq.decode("utf-8")
+            except UnicodeDecodeError:
+                text = f"<0x{byte_seq.hex()}>"
+            self._vocab[text] = token_id
+        
+        # Add special tokens to vocab
+        self._vocab[bos_token] = 1
+        self._vocab[eos_token] = 2
+        self._vocab[pad_token] = 0
+        self._vocab[unk_token] = 3
+        
+        self._id_to_token = {v: k for k, v in self._vocab.items()}
+        
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            unk_token=unk_token,
+            **kwargs,
+        )
+        
+        self.vocab_file = vocab_file
+    
+    @property
+    def vocab_size(self) -> int:
+        return 131072  # Grok-2 vocab size
+    
+    def get_vocab(self):
+        return self._vocab.copy()
+    
+    def _tokenize(self, text: str) -> List[str]:
+        """Tokenize text into subword tokens."""
+        # Simple byte-level tokenization
+        # For production, this should use proper BPE merges
+        tokens = []
+        text_bytes = text.encode("utf-8")
+        
+        i = 0
+        while i < len(text_bytes):
+            # Try to find longest matching byte sequence
+            found = False
+            for length in range(min(20, len(text_bytes) - i), 0, -1):
+                byte_seq = text_bytes[i:i+length]
+                if byte_seq in self._byte_to_token:
+                    token_id = self._byte_to_token[byte_seq]
+                    tokens.append(self._id_to_token.get(token_id, f"<{token_id}>"))
+                    i += length
+                    found = True
+                    break
+            
+            if not found:
+                # Fall back to single byte
+                single_byte = bytes([text_bytes[i]])
+                if single_byte in self._byte_to_token:
+                    token_id = self._byte_to_token[single_byte]
+                    tokens.append(self._id_to_token.get(token_id, f"<{token_id}>"))
+                else:
+                    tokens.append(self.unk_token)
+                i += 1
+        
+        return tokens
+    
+    def _convert_token_to_id(self, token: str) -> int:
+        """Convert token to id."""
+        return self._vocab.get(token, self._vocab.get(self.unk_token, 3))
+    
+    def _convert_id_to_token(self, index: int) -> str:
+        """Convert id to token."""
+        return self._id_to_token.get(index, self.unk_token)
+    
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert tokens back to string."""
+        byte_list = []
+        for token in tokens:
+            if token in self._vocab:
+                token_id = self._vocab[token]
+                if token_id in self._token_to_bytes:
+                    byte_list.extend(self._token_to_bytes[token_id])
+            elif token.startswith("<0x") and token.endswith(">"):
+                # Hex encoded byte
+                try:
+                    hex_str = token[3:-1]
+                    byte_list.extend(bytes.fromhex(hex_str))
+                except ValueError:
+                    pass
+        
+        try:
+            return bytes(byte_list).decode("utf-8", errors="replace")
+        except Exception:
+            return ""
+    
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
+        """Save vocabulary to directory."""
+        import os
+        import shutil
+        
+        if filename_prefix:
+            vocab_file = os.path.join(save_directory, f"{filename_prefix}-tokenizer.tok.json")
+        else:
+            vocab_file = os.path.join(save_directory, "tokenizer.tok.json")
+        
+        if os.path.abspath(self.vocab_file) != os.path.abspath(vocab_file):
+            shutil.copy(self.vocab_file, vocab_file)
+        
+        return (vocab_file,)
+
+
+# For auto registration
+def get_tokenizer_class():
+    return Grok2Tokenizer

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "auto_map": {
+    "AutoTokenizer": "tokenization_grok2.Grok2Tokenizer"
+  },
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|end_of_text|>",
+  "pad_token": "<|pad|>",
+  "unk_token": "<|unk|>",
+  "model_max_length": 131072,
+  "clean_up_tokenization_spaces": false
+}