Hugging Face's logo

Xerv-AI
/
CRAYON-tokenizer

Text Generation
English
generic
tokenizers
hardware-accelerated
avx2
cuda
rocm
double-array-trie
crayon
Model card Files
xet
 Community
CRAYON-tokenizer
File size: 2,963 Bytes
708f4a3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
[build-system]
requires = ["setuptools>=68.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "xerv-crayon"
version = "5.3.6"
description = "Omni-Backend Tokenizer - CPU (AVX2/512), CUDA (NVIDIA), ROCm (AMD) with automatic hardware detection"
readme = "README.md"
requires-python = ">=3.8,<3.13"
license = {file = "LICENSE"}
authors = [
  {name = "Xerv Research Engineering Division", email = "xerv.org@gmail.com"}
]
keywords = [
    "tokenizer", 
    "nlp", 
    "simd", 
    "avx2", 
    "avx512",
    "cuda",
    "rocm",
    "hip",
    "gpu",
    "high-performance", 
    "zero-copy",
    "dat",
    "double-array-trie",
    "machine-learning",
    "deep-learning",
    "transformers",
    "llm",
    "nvcuda",
    "amd",
    "nvidia"
]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Developers",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: C",
    "Programming Language :: C++",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Text Processing :: Linguistic",
    "Operating System :: POSIX :: Linux",
    "Operating System :: Microsoft :: Windows",
    "Operating System :: MacOS",
    "Environment :: GPU :: NVIDIA CUDA",
]

# Core dependencies (minimal for basic usage)
dependencies = []

# Optional dependencies for full functionality
[project.optional-dependencies]
full = [
    "requests>=2.31.0",
    "datasets>=2.18.0",
    "huggingface-hub>=0.21.0"
]
cuda = [
    "torch>=2.0.0",
    "torchvision>=0.15.0", 
    "torchaudio>=2.0.0"
]
dev = [
    "pytest>=7.0.0",
    "pytest-benchmark>=4.0.0",
    "build>=1.0.0",
    "twine>=4.0.0",
    "torch>=2.0.0"
]
benchmark = [
    "tiktoken>=0.5.0",
    "transformers>=4.30.0",
    "matplotlib>=3.7.0"
]

[project.urls]
Homepage = "https://github.com/Electroiscoding/CRAYON"
Repository = "https://github.com/Electroiscoding/CRAYON.git"
Documentation = "https://github.com/Electroiscoding/CRAYON#readme"
"Bug Tracker" = "https://github.com/Electroiscoding/CRAYON/issues"

[project.scripts]
crayon-benchmark = "crayon.cli:run_benchmark"

[tool.setuptools]
package-dir = {"" = "src"}

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools.package-data]
"crayon" = [
    "resources/dat/vocab_lite.dat",
    "resources/dat/vocab_lite.json",
    "resources/dat/vocab_standard.dat",
    "resources/dat/vocab_standard.json",
    "resources/*.txt",
    "resources/*.csv",
    "c_ext/*.h",
    "c_ext/*.c",
    "c_ext/*.cpp",
    "c_ext/*.cu",
    "c_ext/*.hip",
    "c_ext/*.pyd",
    "c_ext/*.so",
    "c_ext/*.py",
    "c_ext/compiled/*.pyd",
    "c_ext/compiled/*.so"
]

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]