| """Prepare tokenized datasets for pretraining.""" | |
| from pathlib import Path | |
| def main() -> None: | |
| output_dir = Path("data/tokenized") | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| print("Add dataset loading and tokenization here.") | |
| if __name__ == "__main__": | |
| main() | |