Update 2 files

- /tokenizer.py
- /trainer.cli.py

tokenizer.py

@@ -147,6 +147,9 @@ class Tokenizer:
 
 
     def c_compile(self):
+        import os
+        import subprocess
+        
         # Get the path of the current Python script
         script_dir = os.path.dirname(__file__)
 

trainer.cli.py

@@ -28,7 +28,7 @@ if __name__ == '__main__':
     dataset = Dataset(config.dataset)
 
     tokenizer = Tokenizer()
-    tokenizer.train(dataset.text, max_length=config.tokenizer.max_length)
+    #tokenizer.train(dataset.text, max_length=config.tokenizer.max_length)
     ids = tokenizer.c_encode(dataset.text)