File size: 1,129 Bytes
eec259f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | """Build the curriculum_tokenizer C++ extension."""
import os
from setuptools import setup, find_packages
from pybind11.setup_helpers import Pybind11Extension, build_ext
if os.name == "nt":
compile_args = ["/O2", "/DNDEBUG"]
else:
compile_args = ["-O3", "-DNDEBUG", "-pthread"]
ext_modules = [
Pybind11Extension(
"curriculum_tokenizer._cpp_tokenizer",
["src/bindings.cpp", "src/tokenizer.cpp"],
include_dirs=["include"],
cxx_std=17,
extra_compile_args=compile_args,
extra_link_args=[] if os.name == "nt" else ["-pthread"],
),
]
setup(
name="bloat-tokenizer",
version="0.3.0",
description="Lossless C++ BPE tokenizer and dataset token-counting CLI",
long_description=open("README.md", encoding="utf-8").read(),
long_description_content_type="text/markdown",
packages=find_packages(where="python"),
package_dir={"": "python"},
ext_modules=ext_modules,
cmdclass={"build_ext": build_ext},
entry_points={"console_scripts": ["bloat-tokenizer=curriculum_tokenizer.cli:main"]},
zip_safe=False,
python_requires=">=3.8",
)
|