Modify: requirements.txt

gramformer_backup/__init__.py

@@ -0,0 +1 @@
+from gramformer.gramformer import Gramformer

gramformer_backup/demo.py

@@ -0,0 +1,30 @@
+from gramformer import Gramformer
+import torch
+
+def set_seed(seed):
+  torch.manual_seed(seed)
+  if torch.cuda.is_available():
+    torch.cuda.manual_seed_all(seed)
+
+set_seed(1212)
+
+
+gf = Gramformer(models = 1, use_gpu=False) # 1=corrector, 2=detector
+
+influent_sentences = [
+    "Matt like fish",
+    "the collection of letters was original used by the ancient Romans",
+    "We enjoys horror movies",
+    "Anna and Mike is going skiing",
+    "I walk to the store and I bought milk",
+    "We all eat the fish and then made dessert",
+    "I will eat fish for dinner and drank milk",
+    "what be the reason for everyone leave the company",
+]   
+
+for influent_sentence in influent_sentences:
+    corrected_sentences = gf.correct(influent_sentence, max_candidates=1)
+    print("[Input] ", influent_sentence)
+    for corrected_sentence in corrected_sentences:
+      print("[Correction] ",corrected_sentence)
+    print("-" *100)

gramformer_backup/gramformer.py

@@ -0,0 +1,127 @@
+class Gramformer:
+
+  def __init__(self, models=1, use_gpu=False):
+    from transformers import AutoTokenizer
+    from transformers import AutoModelForSeq2SeqLM
+    from lm_scorer.models.auto import AutoLMScorer as LMScorer
+    import errant
+    self.annotator = errant.load('en')
+    
+    if use_gpu:
+        device= "cuda:0"
+    else:
+        device = "cpu"
+    batch_size = 1    
+    self.scorer = LMScorer.from_pretrained("gpt2", device=device, batch_size=batch_size)    
+    self.device    = device
+    correction_model_tag = "prithivida/grammar_error_correcter_v1"
+    self.model_loaded = False
+
+    if models == 1:
+        self.correction_tokenizer = AutoTokenizer.from_pretrained(correction_model_tag)
+        self.correction_model     = AutoModelForSeq2SeqLM.from_pretrained(correction_model_tag)
+        self.correction_model     = self.correction_model.to(device)
+        self.model_loaded = True
+        print("[Gramformer] Grammar error correct/highlight model loaded..")
+    elif models == 2:
+        # TODO
+        print("TO BE IMPLEMENTED!!!")
+
+  def correct(self, input_sentence, max_candidates=1):
+      if self.model_loaded:
+        correction_prefix = "gec: "
+        input_sentence = correction_prefix + input_sentence
+        input_ids = self.correction_tokenizer.encode(input_sentence, return_tensors='pt')
+        input_ids = input_ids.to(self.device)
+
+        preds = self.correction_model.generate(
+            input_ids,
+            do_sample=True, 
+            max_length=128, 
+            top_k=50, 
+            top_p=0.95, 
+            early_stopping=True,
+            num_return_sequences=max_candidates)
+
+        corrected = set()
+        for pred in preds:  
+          corrected.add(self.correction_tokenizer.decode(pred, skip_special_tokens=True).strip())
+
+        corrected = list(corrected)
+        scores = self.scorer.sentence_score(corrected, log=True)
+        ranked_corrected = [(c,s) for c, s in zip(corrected, scores)]
+        ranked_corrected.sort(key = lambda x:x[1], reverse=True)
+        return ranked_corrected
+      else:
+        print("Model is not loaded")  
+        return None
+
+  def highlight(self, orig, cor):
+      edits = self._get_edits(orig, cor)
+      orig_tokens = orig.split()
+
+      ignore_indexes = []
+
+      for edit in edits:
+          edit_type = edit[0]
+          edit_str_start = edit[1]
+          edit_spos = edit[2]
+          edit_epos = edit[3]
+          edit_str_end = edit[4]
+
+          # if no_of_tokens(edit_str_start) > 1 ==> excluding the first token, mark all other tokens for deletion
+          for i in range(edit_spos+1, edit_epos):
+            ignore_indexes.append(i)
+
+          if edit_str_start == "":
+              if edit_spos - 1 >= 0:
+                  new_edit_str = orig_tokens[edit_spos - 1]
+                  edit_spos -= 1
+              else:
+                  new_edit_str = orig_tokens[edit_spos + 1]
+                  edit_spos += 1
+              if edit_type == "PUNCT":
+                st = "<a type='" + edit_type + "' edit='" + \
+                    edit_str_end + "'>" + new_edit_str + "</a>"
+              else:
+                st = "<a type='" + edit_type + "' edit='" + new_edit_str + \
+                    " " + edit_str_end + "'>" + new_edit_str + "</a>"
+              orig_tokens[edit_spos] = st
+          elif edit_str_end == "":
+            st = "<d type='" + edit_type + "' edit=''>" + edit_str_start + "</d>"
+            orig_tokens[edit_spos] = st
+          else:
+            st = "<c type='" + edit_type + "' edit='" + \
+                edit_str_end + "'>" + edit_str_start + "</c>"
+            orig_tokens[edit_spos] = st
+
+      for i in sorted(ignore_indexes, reverse=True):
+        del(orig_tokens[i])
+
+      return(" ".join(orig_tokens))
+
+  def detect(self, input_sentence):
+        # TO BE IMPLEMENTED
+        pass
+
+  def _get_edits(self, orig, cor):
+        orig = self.annotator.parse(orig)
+        cor = self.annotator.parse(cor)
+        alignment = self.annotator.align(orig, cor)
+        edits = self.annotator.merge(alignment)
+
+        if len(edits) == 0:  
+            return []
+
+        edit_annotations = []
+        for e in edits:
+            e = self.annotator.classify(e)
+            edit_annotations.append((e.type[2:], e.o_str, e.o_start, e.o_end,  e.c_str, e.c_start, e.c_end))
+                
+        if len(edit_annotations) > 0:
+            return edit_annotations
+        else:    
+            return []
+
+  def get_edits(self, orig, cor):
+      return self._get_edits(orig, cor)

requirements.txt

@@ -1,4 +1,10 @@
-pip==20.1.1
 st-annotated-text
 beautifulsoup4
-git+https://github.com/PrithivirajDamodaran/Gramformer.git
+transformers
+sentencepiece==0.1.95
+python-Levenshtein==0.12.2
+fuzzywuzzy==0.18.0
+tokenizers==0.10.2
+fsspec==2021.5.0
+lm-scorer==0.4.2
+errant

setup.py

@@ -0,0 +1,19 @@
+import setuptools
+
+setuptools.setup(
+    name="gramformer",
+    version="1.0",
+    author="prithiviraj damodaran",
+    author_email="",
+    description="Gramformer",
+    long_description="A framework for detecting, highlighting and correcting grammatical errors on natural language text",
+    url="https://github.com/PrithivirajDamodaran/Gramformer.git",
+    packages=setuptools.find_packages(),
+    install_requires=['transformers', 'sentencepiece==0.1.95', 'python-Levenshtein==0.12.2', 'fuzzywuzzy==0.18.0',  'tokenizers==0.10.2', 'fsspec==2021.5.0', 'lm-scorer==0.4.2', 'errant', 'st-annotated-text'],
+    classifiers=[
+        "Programming Language :: Python :: 3.7",
+        "License :: Apache 2.0",
+        "Operating System :: OS Independent",
+    ],
+)
+