Nicholas Bumgarner commited on
Commit
0ace216
·
1 Parent(s): 4aa8326

Fix tokenizer get_vocab API call

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -74,7 +74,7 @@ def load_tokenizer():
74
  from tokenizers import Tokenizer
75
  _tokenizer = Tokenizer.from_file(tok_path)
76
  # Build a simple vocab-size attribute for compatibility
77
- _tokenizer.vocab_size = len(_tokenizer.get_vocab(with_added=False))
78
  print(f"[Phox Space] Tokenizer loaded (vocab={_tokenizer.vocab_size})")
79
  except Exception as e:
80
  print(f"[Phox Space] Tokenizer download failed: {e}")
@@ -86,7 +86,7 @@ def load_tokenizer():
86
  try:
87
  from tokenizers import Tokenizer
88
  _tokenizer = Tokenizer.from_file(tok_path)
89
- _tokenizer.vocab_size = len(_tokenizer.get_vocab(with_added=False))
90
  print(f"[Phox Space] Tokenizer loaded from {tok_path}")
91
  break
92
  except Exception:
 
74
  from tokenizers import Tokenizer
75
  _tokenizer = Tokenizer.from_file(tok_path)
76
  # Build a simple vocab-size attribute for compatibility
77
+ _tokenizer.vocab_size = len(_tokenizer.get_vocab())
78
  print(f"[Phox Space] Tokenizer loaded (vocab={_tokenizer.vocab_size})")
79
  except Exception as e:
80
  print(f"[Phox Space] Tokenizer download failed: {e}")
 
86
  try:
87
  from tokenizers import Tokenizer
88
  _tokenizer = Tokenizer.from_file(tok_path)
89
+ _tokenizer.vocab_size = len(_tokenizer.get_vocab())
90
  print(f"[Phox Space] Tokenizer loaded from {tok_path}")
91
  break
92
  except Exception: