Spaces:
Sleeping
Sleeping
github-actions[bot]
commited on
Commit
·
e772457
1
Parent(s):
817f16e
Auto-sync from demo at Tue Sep 30 08:47:50 UTC 2025
Browse files
graphgen/models/tokenizer/__init__.py
CHANGED
|
@@ -3,7 +3,6 @@ from typing import List
|
|
| 3 |
|
| 4 |
from graphgen.bases import BaseTokenizer
|
| 5 |
|
| 6 |
-
from .hf_tokenizer import HFTokenizer
|
| 7 |
from .tiktoken_tokenizer import TiktokenTokenizer
|
| 8 |
|
| 9 |
try:
|
|
@@ -22,6 +21,8 @@ def get_tokenizer_impl(tokenizer_name: str = "cl100k_base") -> BaseTokenizer:
|
|
| 22 |
|
| 23 |
# 2. HuggingFace
|
| 24 |
if _HF_AVAILABLE:
|
|
|
|
|
|
|
| 25 |
return HFTokenizer(model_name=tokenizer_name)
|
| 26 |
|
| 27 |
raise ValueError(
|
|
|
|
| 3 |
|
| 4 |
from graphgen.bases import BaseTokenizer
|
| 5 |
|
|
|
|
| 6 |
from .tiktoken_tokenizer import TiktokenTokenizer
|
| 7 |
|
| 8 |
try:
|
|
|
|
| 21 |
|
| 22 |
# 2. HuggingFace
|
| 23 |
if _HF_AVAILABLE:
|
| 24 |
+
from .hf_tokenizer import HFTokenizer
|
| 25 |
+
|
| 26 |
return HFTokenizer(model_name=tokenizer_name)
|
| 27 |
|
| 28 |
raise ValueError(
|