| # requirements.txt | |
| # Core dependencies | |
| jsonlines | |
| click | |
| filelock | |
| pysimdjson; python_version >= "3.9" and python_version < "3.13" | |
| # Editable install for tokenizer | |
| -e tokenizers_superbpe/bindings/python/ | |
| # requirements.txt | |
| # Core dependencies | |
| jsonlines | |
| click | |
| filelock | |
| pysimdjson; python_version >= "3.9" and python_version < "3.13" | |
| # Editable install for tokenizer | |
| -e tokenizers_superbpe/bindings/python/ | |