PeteBleackley commited on
Commit
fb4c0b0
·
1 Parent(s): 519dfd1

Minor error in setting up tokenizer

Browse files
Files changed (1) hide show
  1. scripts.py +1 -1
scripts.py CHANGED
@@ -123,7 +123,7 @@ def prepare_training_datasets():
123
  consistency.to_csv('corpora/consistency.csv')
124
 
125
  def train_models(path):
126
- tokenizer = tokenizers.from_pretrained('roberta-base')
127
  trainer = qarac.models.QaracTrainerModel.QaracTrainerModel('roberta_base',
128
  tokenizer)
129
  loss_fn = CombinedLoss()
 
123
  consistency.to_csv('corpora/consistency.csv')
124
 
125
  def train_models(path):
126
+ tokenizer = tokenizers.Tokenizer.from_pretrained('roberta-base')
127
  trainer = qarac.models.QaracTrainerModel.QaracTrainerModel('roberta_base',
128
  tokenizer)
129
  loss_fn = CombinedLoss()