File size: 1,029 Bytes
4265aea | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """EZ-Tokenizer - High-performance Python code tokenizer with adaptive resource management.
Features:
- Efficient tokenization of code and text
- Adaptive resource management
- Support for large datasets
- Custom vocabulary generation
"""
__version__ = "1.0.0"
__author__ = "EZ-Tokenizer Team"
__all__ = [
"SystemResources",
"log_memory_usage",
"manage_ram",
"build_tokenizer"
]
# Lazy imports to prevent circular imports
def __getattr__(name):
if name == 'SystemResources':
from .resources import SystemResources
return SystemResources
elif name in ('log_memory_usage', 'manage_ram', 'build_tokenizer'):
from .adaptive_tokenizer import log_memory_usage, manage_ram, build_tokenizer
if name == 'log_memory_usage':
return log_memory_usage
elif name == 'manage_ram':
return manage_ram
elif name == 'build_tokenizer':
return build_tokenizer
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|