Spaces:
Sleeping
Sleeping
File size: 1,619 Bytes
dbc6675 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | def create_tokenizer(name: str, **kwargs):
"""Factory function to create a tokenizer by name."""
normalized = "wl" if name == "graphs" else name
if normalized == "wl":
# Import inside to avoid hard dependency if wlplan is missing
# and user is using other tokenizers
try:
from code.tokenization.wl import WLTokenizer
return WLTokenizer(iterations=kwargs.get("iterations", 2))
except ImportError:
raise ImportError("WLTokenizer requires 'wlplan' package.")
elif normalized == "simhash":
from code.tokenization.simhash import SimHashTokenizer
return SimHashTokenizer(
hash_dim=kwargs.get("hash_dim", 128),
seed=kwargs.get("seed", 42),
)
elif normalized == "shortest_path":
from code.tokenization.shortest_path import ShortestPathTokenizer
return ShortestPathTokenizer(
max_path_length=kwargs.get("max_path_length", 5),
)
elif normalized == "graphbpe":
from code.tokenization.graphbpe import GraphBPETokenizer
return GraphBPETokenizer(
vocab_size=kwargs.get("vocab_size", 1000),
num_iterations=kwargs.get("num_iterations", 100),
)
elif normalized == "random":
from code.tokenization.random import RandomTokenizer
return RandomTokenizer(
random_dim=kwargs.get("random_dim", 128),
seed=kwargs.get("seed", 42),
normalize=kwargs.get("normalize", True),
)
else:
raise ValueError(f"Unknown tokenizer: {name}")
|