Fix: Use trust_remote_code=True for OpenLLM custom tokenizer classes

app.py

@@ -110,56 +110,24 @@ class OpenLLMTrainer:
             
             model_name = model_mapping.get(model_size, "lemms/openllm-small-extended-7k")
             
-            # Try multiple approaches to load the tokenizer
-            tokenizer_loaded = False
-            
-            # Approach 1: Try direct loading with trust_remote_code
+            # Load OpenLLM custom tokenizer with trust_remote_code
             try:
-                print("🔄 Attempting to load tokenizer with trust_remote_code=True...")
+                print("🔄 Loading OpenLLM custom tokenizer...")
                 self.tokenizer = AutoTokenizer.from_pretrained(
                     model_name,
-                    trust_remote_code=True,
-                    use_fast=False  # Use slow tokenizer as fallback
+                    trust_remote_code=True,  # CRITICAL for OpenLLM custom tokenizer classes
+                    use_fast=False          # Use slow tokenizer for compatibility
                 )
-                tokenizer_loaded = True
-                print("✅ Tokenizer loaded with trust_remote_code=True")
-            except Exception as e1:
-                print(f"❌ Approach 1 failed: {e1}")
+                print(f"✅ OpenLLM custom tokenizer loaded: {type(self.tokenizer).__name__}")
                 
-                # Approach 2: Try with use_fast=False
-                try:
-                    print("🔄 Attempting to load tokenizer with use_fast=False...")
-                    self.tokenizer = AutoTokenizer.from_pretrained(
-                        model_name,
-                        use_fast=False
-                    )
-                    tokenizer_loaded = True
-                    print("✅ Tokenizer loaded with use_fast=False")
-                except Exception as e2:
-                    print(f"❌ Approach 2 failed: {e2}")
+                # Add padding token if not present
+                if self.tokenizer.pad_token is None:
+                    self.tokenizer.pad_token = self.tokenizer.eos_token
+                    print("✅ Added padding token")
                     
-                    # Approach 3: Try with legacy tokenizer
-                    try:
-                        print("🔄 Attempting to load tokenizer with legacy settings...")
-                        self.tokenizer = AutoTokenizer.from_pretrained(
-                            model_name,
-                            use_fast=False,
-                            legacy=True
-                        )
-                        tokenizer_loaded = True
-                        print("✅ Tokenizer loaded with legacy settings")
-                    except Exception as e3:
-                        print(f"❌ Approach 3 failed: {e3}")
-                        
-                        # Approach 4: Try loading from a different model as fallback
-                        try:
-                            print("🔄 Attempting to load fallback tokenizer...")
-                            self.tokenizer = AutoTokenizer.from_pretrained("gpt2")
-                            tokenizer_loaded = True
-                            print("✅ Fallback tokenizer loaded (GPT-2)")
-                        except Exception as e4:
-                            print(f"❌ All tokenizer loading approaches failed")
-                            return f"❌ Failed to load any tokenizer: {str(e4)}"
+            except Exception as e:
+                print(f"❌ Failed to load OpenLLM custom tokenizer: {e}")
+                return f"❌ Failed to load OpenLLM custom tokenizer: {str(e)}"
             
             # Add padding token if not present
             if self.tokenizer.pad_token is None: