File size: 497 Bytes
41788c4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | from docsifer.core.tokenizer import TiktokenCounter
def test_counts_zero_for_empty() -> None:
assert TiktokenCounter("gpt-4o").count("") == 0
def test_counts_positive_for_text() -> None:
counter = TiktokenCounter("gpt-4o")
assert counter.count("Hello world!") > 0
def test_unknown_model_falls_back() -> None:
# Should not raise; falls back to cl100k_base or whitespace.
counter = TiktokenCounter("definitely-not-a-real-model")
assert counter.count("hello world") > 0
|