Update README.md
Browse files
README.md
CHANGED
|
@@ -22,7 +22,7 @@ After downloading the dataset, we went on the way to mask LM.
|
|
| 22 |
|
| 23 |
```py
|
| 24 |
from transformers import AutoTokenizer, AutoModel
|
| 25 |
-
bnbert_tokenizer = AutoTokenizer.from_pretrained("Kowsher/bert-base-
|
| 26 |
text = "খাঁটি সোনার চাইতে খাঁটি আমার দেশের মাটি"
|
| 27 |
bnbert_tokenizer.tokenize(text)
|
| 28 |
# output: ['খাটি', 'সে', '##ানার', 'চাইতে', 'খাটি', 'আমার', 'দেশের', 'মাটি']
|
|
@@ -31,8 +31,8 @@ bnbert_tokenizer.tokenize(text)
|
|
| 31 |
here, we can use bert base bangla model as for masked language modeling:
|
| 32 |
```py
|
| 33 |
from transformers import BertForMaskedLM, BertTokenizer, pipeline
|
| 34 |
-
model = BertForMaskedLM.from_pretrained("Kowsher/bert-base-
|
| 35 |
-
tokenizer = BertTokenizer.from_pretrained("Kowsher/bert-base-
|
| 36 |
|
| 37 |
nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer)
|
| 38 |
for pred in nlp(f"আমি বাংলার গান {nlp.tokenizer.mask_token}"):
|
|
|
|
| 22 |
|
| 23 |
```py
|
| 24 |
from transformers import AutoTokenizer, AutoModel
|
| 25 |
+
bnbert_tokenizer = AutoTokenizer.from_pretrained("Kowsher/bert-base-bangla")
|
| 26 |
text = "খাঁটি সোনার চাইতে খাঁটি আমার দেশের মাটি"
|
| 27 |
bnbert_tokenizer.tokenize(text)
|
| 28 |
# output: ['খাটি', 'সে', '##ানার', 'চাইতে', 'খাটি', 'আমার', 'দেশের', 'মাটি']
|
|
|
|
| 31 |
here, we can use bert base bangla model as for masked language modeling:
|
| 32 |
```py
|
| 33 |
from transformers import BertForMaskedLM, BertTokenizer, pipeline
|
| 34 |
+
model = BertForMaskedLM.from_pretrained("Kowsher/bert-base-bangla")
|
| 35 |
+
tokenizer = BertTokenizer.from_pretrained("Kowsher/bert-base-bangla")
|
| 36 |
|
| 37 |
nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer)
|
| 38 |
for pred in nlp(f"আমি বাংলার গান {nlp.tokenizer.mask_token}"):
|