File size: 1,545 Bytes
d61821a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | """Exact token accounting with the frozen local Qwen tokenizer."""
from __future__ import annotations
from hashlib import sha256
from pathlib import Path
from typing import Sequence
from tokenizers import Tokenizer
from .components import Candidate
DEFAULT_TOKENIZER = (
Path.home()
/ ".lmstudio/models/lmstudio-community/"
"Qwen3.6-35B-A3B-MLX-4bit/tokenizer.json"
)
class QwenTokenCounter:
def __init__(self, path: Path = DEFAULT_TOKENIZER):
self.path = path.resolve()
raw = self.path.read_bytes()
self.sha256 = sha256(raw).hexdigest()
self.tokenizer = Tokenizer.from_file(str(self.path))
def count(self, text: str) -> int:
return len(self.tokenizer.encode(text, add_special_tokens=False).ids)
def pack_ranked(
self,
candidates: Sequence[Candidate],
budget: int,
) -> tuple[str, tuple[Candidate, ...], int]:
blocks: list[str] = []
included: list[Candidate] = []
used = 0
for rank, candidate in enumerate(candidates, start=1):
block = (
f"\n--- Rank {rank}: {candidate.path} "
f"(lines {candidate.line_start}-{candidate.line_end}; {candidate.source}) ---\n"
f"{candidate.text}\n"
)
tokens = self.count(block)
if used + tokens > budget:
continue
blocks.append(block)
included.append(candidate)
used += tokens
return "".join(blocks), tuple(included), used
|