Spaces:
Running
Running
Commit
·
84a67cd
1
Parent(s):
6520201
Update bert_ner_model_loader.py
Browse files- bert_ner_model_loader.py +2 -0
bert_ner_model_loader.py
CHANGED
|
@@ -11,6 +11,8 @@ from torch.nn import CrossEntropyLoss
|
|
| 11 |
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
|
| 12 |
from torch.utils.data.distributed import DistributedSampler
|
| 13 |
from tqdm import tqdm, trange
|
|
|
|
|
|
|
| 14 |
from nltk import word_tokenize
|
| 15 |
# from transformers import (BertConfig, BertForTokenClassification,
|
| 16 |
# BertTokenizer)
|
|
|
|
| 11 |
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
|
| 12 |
from torch.utils.data.distributed import DistributedSampler
|
| 13 |
from tqdm import tqdm, trange
|
| 14 |
+
import nltk
|
| 15 |
+
nltk.download('punkt')
|
| 16 |
from nltk import word_tokenize
|
| 17 |
# from transformers import (BertConfig, BertForTokenClassification,
|
| 18 |
# BertTokenizer)
|