Update README.md

README.md

@@ -25,7 +25,7 @@ Concept:
 This will be trained for 2 epochs. The current model here is the first one.
 
 ```python
-import torch
+import torch, re, unicodedata
 from transformers import AutoModel, AutoTokenizer
 
 model_name = "cnmoro/LexicalEmbed-Base"
@@ -34,7 +34,14 @@ tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
 model.eval()
 
+def preprocess(text):
+    text = unicodedata.normalize('NFD', text)
+    text = ''.join(c for c in text if unicodedata.category(c) != 'Mn')
+    text = re.sub(r'[^\w\s]+', ' ', text.lower())
+    return re.sub(r'\s+', ' ', text).strip()
+
 texts = ["hello world", "hel wor"]
+texts = [ preprocess(s) for s in texts ]
 inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
 
 with torch.no_grad():