Spaces:
Paused
Paused
Nicholas Bumgarner commited on
Commit ·
0ace216
1
Parent(s): 4aa8326
Fix tokenizer get_vocab API call
Browse files
app.py
CHANGED
|
@@ -74,7 +74,7 @@ def load_tokenizer():
|
|
| 74 |
from tokenizers import Tokenizer
|
| 75 |
_tokenizer = Tokenizer.from_file(tok_path)
|
| 76 |
# Build a simple vocab-size attribute for compatibility
|
| 77 |
-
_tokenizer.vocab_size = len(_tokenizer.get_vocab(
|
| 78 |
print(f"[Phox Space] Tokenizer loaded (vocab={_tokenizer.vocab_size})")
|
| 79 |
except Exception as e:
|
| 80 |
print(f"[Phox Space] Tokenizer download failed: {e}")
|
|
@@ -86,7 +86,7 @@ def load_tokenizer():
|
|
| 86 |
try:
|
| 87 |
from tokenizers import Tokenizer
|
| 88 |
_tokenizer = Tokenizer.from_file(tok_path)
|
| 89 |
-
_tokenizer.vocab_size = len(_tokenizer.get_vocab(
|
| 90 |
print(f"[Phox Space] Tokenizer loaded from {tok_path}")
|
| 91 |
break
|
| 92 |
except Exception:
|
|
|
|
| 74 |
from tokenizers import Tokenizer
|
| 75 |
_tokenizer = Tokenizer.from_file(tok_path)
|
| 76 |
# Build a simple vocab-size attribute for compatibility
|
| 77 |
+
_tokenizer.vocab_size = len(_tokenizer.get_vocab())
|
| 78 |
print(f"[Phox Space] Tokenizer loaded (vocab={_tokenizer.vocab_size})")
|
| 79 |
except Exception as e:
|
| 80 |
print(f"[Phox Space] Tokenizer download failed: {e}")
|
|
|
|
| 86 |
try:
|
| 87 |
from tokenizers import Tokenizer
|
| 88 |
_tokenizer = Tokenizer.from_file(tok_path)
|
| 89 |
+
_tokenizer.vocab_size = len(_tokenizer.get_vocab())
|
| 90 |
print(f"[Phox Space] Tokenizer loaded from {tok_path}")
|
| 91 |
break
|
| 92 |
except Exception:
|