File size: 1,324 Bytes
4c57a1d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# Test Code
```python
import tensorflow as tf
from transformers import TFAutoModelForPreTraining, AutoTokenizer
from normalizer import normalize
import numpy as np

model = TFAutoModelForPreTraining.from_pretrained("SarwarShafee/BanglaBert_with_TFModel", from_pt=True)
tokenizer = AutoTokenizer.from_pretrained("SarwarShafee/BanglaBert_with_TFModel")

original_sentence = "আমি কৃতজ্ঞ কারণ আপনি আমার জন্য অনেক কিছু করেছেন।"
fake_sentence = "আমি হতাশ কারণ আপনি আমার জন্য অনেক কিছু করেছেন।"
fake_sentence = normalize(fake_sentence)  # this normalization step is required before tokenizing the text

fake_tokens = tokenizer.tokenize(fake_sentence)
fake_inputs = tokenizer.encode(fake_sentence, return_tensors="tf")
discriminator_outputs = model(fake_inputs)[0]
predictions = tf.round((tf.sign(discriminator_outputs) + 1) / 2)

# Convert the predictions to a Python list and then to integers
predictions_list = predictions.numpy().squeeze().tolist()
integer_predictions = [int(prediction[0]) for prediction in predictions_list[1:-1]]

print(" ".join(fake_tokens))
print("-" * 50)
print(" ".join([str(prediction) for prediction in integer_predictions]))
print("-" * 50)

```