🔧 Fix tokenizer crash with unknown custom models

- Add model-specific encoder fallback for unknown models
- Graceful error handling when tiktoken fails on custom models
- Use character fallback when tokenization fails
- Fix crash when using custom models like o3-mini

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

ai/tokenizers/openai_tokenizer.py

@@ -24,6 +24,18 @@ class OpenAITokenizer:
                 except:
                     pass
     
+    def _get_encoder_for_model(self, model: str):
+        """Get the appropriate encoder for a model, with fallback for unknown models."""
+        if not TIKTOKEN_AVAILABLE:
+            return None
+            
+        try:
+            # Try to get model-specific encoder
+            return tiktoken.encoding_for_model(model)
+        except:
+            # Fallback to default encoder if model is unknown
+            return self.encoder
+    
     def count_tokens(self, prompt: Union[str, List[Dict[str, Any]]], model: str = "gpt-3.5-turbo") -> Dict[str, Any]:
         """Count tokens for OpenAI request with proper formatting."""
         start_time = time.time()
@@ -58,7 +70,10 @@ class OpenAITokenizer:
     
     def _count_chat_tokens(self, messages: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
         """Count tokens for chat messages with proper OpenAI formatting."""
-        if not self.encoder:
+        # Get model-specific encoder with fallback
+        encoder = self._get_encoder_for_model(model)
+        
+        if not encoder:
             # Fallback for chat messages
             total_chars = sum(len(str(msg.get("content", ""))) for msg in messages)
             return {
@@ -99,7 +114,11 @@ class OpenAITokenizer:
                 if len(text_content) > 800000 or num_tokens > 200000:
                     raise ValueError("Content is too large to tokenize.")
                 
-                num_tokens += len(self.encoder.encode(text_content))
+                try:
+                    num_tokens += len(encoder.encode(text_content))
+                except Exception as e:
+                    # Fallback to character count if tokenization fails
+                    num_tokens += len(text_content) // 4
                 
                 if key == "name":
                     num_tokens += tokens_per_name