| """Deduplicate cleaned text.""" | |
| from pathlib import Path | |
| def main() -> None: | |
| input_dir = Path("data/cleaned") | |
| output_dir = Path("data/deduplicated") | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| print(f"Add deduplication logic for {input_dir} -> {output_dir}") | |
| if __name__ == "__main__": | |
| main() | |