Fix: Store tokenizer path separately to avoid SentencePieceProcessor attribute error

app.py

@@ -144,11 +144,13 @@ class OpenLLMTrainer:
                 sp_processor = spm.SentencePieceProcessor()
                 sp_processor.load(tokenizer_path)
                 
-                # Store tokenizer for later use
+                # Store tokenizer and its path separately
                 self.tokenizer = sp_processor
+                self.tokenizer_path = tokenizer_path  # Store the path separately
                 
                 print(f"✅ Tokenizer loaded successfully using SentencePieceProcessor")
                 print(f"   Vocabulary size: {sp_processor.vocab_size()}")
+                print(f"   Tokenizer path: {tokenizer_path}")
                 
             except Exception as e:
                 print(f"❌ Failed to load tokenizer: {e}")
@@ -186,8 +188,8 @@ class OpenLLMTrainer:
             
             # Create OpenLLM's TextDataLoader
             try:
-                # Get tokenizer path
-                tokenizer_path = self.tokenizer.model_file_path
+                # Use the stored tokenizer path instead of trying to access model_file_path
+                tokenizer_path = self.tokenizer_path  # Use the stored path
                 
                 self.data_loader = TextDataLoader(
                     data_file=temp_data_file,
@@ -407,9 +409,9 @@ class OpenLLMTrainer:
             tokenizer_dir = os.path.join(config.output_dir, "tokenizer")
             os.makedirs(tokenizer_dir, exist_ok=True)
             
-            # Copy the tokenizer.model file
+            # Copy the tokenizer.model file using the stored path
             import shutil
-            shutil.copy2(self.tokenizer.model_file_path, os.path.join(tokenizer_dir, "tokenizer.model"))
+            shutil.copy2(self.tokenizer_path, os.path.join(tokenizer_dir, "tokenizer.model"))
             
             print("✅ Model saved locally")