Add n-gram update

main.py

@@ -25,15 +25,16 @@ def add_dict(a, b):
     return temp
 
 class Chatbot:
-    def __init__(self, name=None, letter_replace: bool = True, data: dict = None, frequency_weight: float = 0, div_by_len: bool = False):
+    def __init__(self, name = None, n: int = 1, letter_replace: bool = True, data: dict = None, frequency_weight: float = 0, div_by_len: bool = False):
         self.name = name
         self.letter_replace = letter_replace
         self.frequency_weight = frequency_weight
         self.div_by_len = div_by_len
         self.model = {}
+        self.n = n-1
         if data is not None:
             self.train(data)
-    def tokenize(self, text: str):
+    def tokenize(self, text: str, n: int = 1):
         preprocess = ""
         for x in text.lower():
             if x in letters:
@@ -42,8 +43,13 @@ class Chatbot:
                 else:
                     preprocess += x
             else:
-                preprocess += " "+x+" "
-        return preprocess.split()
+                preprocess += " " + x + " "
+        tokens = preprocess.split()
+        output = tokens.copy()
+        for i in range(self.n):
+            for num, word in enumerate(tokens[:-i]):
+                output.append(' '.join(tokens[num:num+i]))
+        return output
     def train(self, data: dict):
         lendata = len(data)
         lendata_div = 1/lendata