check if word is plural

utils.py

@@ -1,4 +1,7 @@
 import torch
+import nltk
+from nltk import pos_tag
+from nltk.tokenize import word_tokenize
 from transformers.models.bert.modeling_bert import BertModel, BertPreTrainedModel
 from torch import nn
 from itertools import chain
@@ -9,6 +12,9 @@ import re
 import string
 import inflect
 
+nltk.download("averaged_perceptron_tagger")
+nltk.download("punkt")
+
 punct_chars = list((set(string.punctuation) | {'’', '‘', '–', '—', '~', '|', '“', '”', '…', "'", "`", '_'}))
 punct_chars.sort()
 punctuation = ''.join(punct_chars)
@@ -825,6 +831,15 @@ PLURAL_TO_SINGULAR_EXCLUSIONS = [
 
 p = inflect.engine()
 
+def is_plural(word):
+    """Determine if a word is plural using NLTK's part-of-speech tagging."""
+    # Tokenize the input word (necessary for NLTK tagging)
+    tokens = word_tokenize(word)
+    # Get the part-of-speech tag for the word
+    pos = pos_tag(tokens)[0][1]
+    # Check if the word is tagged as plural (NNS or NNPS in Penn Treebank tags)
+    return pos in ["NNS", "NNPS"]
+
 def singular_to_plural(word):
     """Convert singular words to plural using inflect."""
     plural = p.plural(word)
@@ -832,9 +847,9 @@ def singular_to_plural(word):
 
 def plural_to_singular(word):
     """Convert plural word to singular using inflect."""
-    if word in PLURAL_TO_SINGULAR_EXCLUSIONS:
-        return word
-    return p.singular_noun(word) or word 
+    if is_plural(word):
+        return p.singular_noun(word) or word 
+    return word
 
 plural_MATH_WORDS = [singular_to_plural(word) for word in MATH_WORDS]