StentorLabs commited on
Commit
ac7b750
·
verified ·
1 Parent(s): 8be2e27

Update load_stentor2.py

Browse files
Files changed (1) hide show
  1. load_stentor2.py +2 -0
load_stentor2.py CHANGED
@@ -9,6 +9,8 @@ MODEL_ID = "StentorLabs/Stentor2-12M-Preview"
9
 
10
  class _Tokenizer:
11
  def __init__(self, vocab_path):
 
 
12
  import tokenmonster
13
  self._vocab = tokenmonster.load(vocab_path)
14
  self._vocab.add_special_token("</s>")
 
9
 
10
  class _Tokenizer:
11
  def __init__(self, vocab_path):
12
+ import subprocess, sys
13
+ subprocess.run([sys.executable, "-m", "pip", "install", "tokenmonster", "-q"], check=True)
14
  import tokenmonster
15
  self._vocab = tokenmonster.load(vocab_path)
16
  self._vocab.add_special_token("</s>")