VinayNR commited on
Commit
8a8dcbd
·
1 Parent(s): 3950984

Update tokenizer.py

Browse files
Files changed (1) hide show
  1. tokenizer.py +3 -3
tokenizer.py CHANGED
@@ -1,14 +1,14 @@
1
- from flair.data import Sentence, Tokenizer
2
  from typing import List
3
 
4
 
5
  class StatsTokenizer(Tokenizer):
6
 
7
  def __init__(self):
8
- super(CustomTokenizer, self).__init__()
9
 
10
  def tokenize(self, text: str) -> List[str]:
11
- return CustomTokenizer.run_tokenize(text)
12
 
13
  @staticmethod
14
  def run_tokenize(text: str) -> List[str]:
 
1
+ from flair.data import Tokenizer
2
  from typing import List
3
 
4
 
5
  class StatsTokenizer(Tokenizer):
6
 
7
  def __init__(self):
8
+ super(StatsTokenizer, self).__init__()
9
 
10
  def tokenize(self, text: str) -> List[str]:
11
+ return StatsTokenizer.run_tokenize(text)
12
 
13
  @staticmethod
14
  def run_tokenize(text: str) -> List[str]: