github-actions[bot] commited on
Commit
e772457
·
1 Parent(s): 817f16e

Auto-sync from demo at Tue Sep 30 08:47:50 UTC 2025

Browse files
graphgen/models/tokenizer/__init__.py CHANGED
@@ -3,7 +3,6 @@ from typing import List
3
 
4
  from graphgen.bases import BaseTokenizer
5
 
6
- from .hf_tokenizer import HFTokenizer
7
  from .tiktoken_tokenizer import TiktokenTokenizer
8
 
9
  try:
@@ -22,6 +21,8 @@ def get_tokenizer_impl(tokenizer_name: str = "cl100k_base") -> BaseTokenizer:
22
 
23
  # 2. HuggingFace
24
  if _HF_AVAILABLE:
 
 
25
  return HFTokenizer(model_name=tokenizer_name)
26
 
27
  raise ValueError(
 
3
 
4
  from graphgen.bases import BaseTokenizer
5
 
 
6
  from .tiktoken_tokenizer import TiktokenTokenizer
7
 
8
  try:
 
21
 
22
  # 2. HuggingFace
23
  if _HF_AVAILABLE:
24
+ from .hf_tokenizer import HFTokenizer
25
+
26
  return HFTokenizer(model_name=tokenizer_name)
27
 
28
  raise ValueError(