Update README.md

README.md

@@ -17,15 +17,27 @@ via Contrastive Fine-Tuning of mMiniLMv2 without References
 ## How to use
 
 ```python
+import re
 from transformers import AutoTokenizer, AutoModel
 
+def preprocess(text: str):
+  text = text.lower()
+  text = re.sub(r'[\(\[].*?[\)\]]', '', text)
+  text = re.sub(r'[^\w\s]', '', text)
+  return text
+
 tokenizer = AutoTokenizer.from_pretrained("aixplain/NoRef-ER")
 model = AutoModel.from_pretrained("aixplain/NoRef-ER")
 
-tokens = tokenizer([
+# preprocess
+texts = [
     "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced.",
     "In Italy, pizzas serves in formal settings, such as at an restaurant, is presented unslicing."
-], padding=True, return_tensors="pt")
+]
+preprocessed_texts = [preprocess(text) for text in texts]
+# tokenize
+tokens = tokenizer(preprocessed_texts, padding=True, return_tensors="pt")
+# score
 scores = model.score(**tokens)
 ```