Upload main.py

main.py

@@ -0,0 +1,244 @@
+import numpy as np
+from numba import njit
+from tqdm import tqdm
+import math
+import random
+from matplotlib import pyplot as plt
+import pickle
+
+
+# whitelist = "ёйцукенгшщзхъфывапролджэячсмитьбю "
+
+def text_to_arr(text: str):
+    return np.array([ord(x) for x in text.lower()])
+
+@njit
+def longest_common_substring(s1, s2):
+    current_match_start = -1
+    current_match_end = -1
+
+    best_match_start = current_match_start
+    best_match_end = current_match_end
+
+    min_len = min(len(s1), len(s2))
+    for i in range(min_len):
+        if s1[i] == s2[i]:
+            current_match_start = current_match_end = i
+            j = 0
+            while s1[i+j] == s2[i+j] and i+j < min_len:
+               j += 1
+            current_match_end = current_match_start + j
+
+            if current_match_end - current_match_start > best_match_end - best_match_start:
+                best_match_start = current_match_start
+                best_match_end = current_match_end
+
+    return s1[best_match_start:best_match_end]
+
+def not_found_in(q, data):
+    for l in data:
+        count = 0
+        lq = len(q)-1
+        for v in l:
+            if v == q[count]:
+                count += 1
+            else:
+                count = 0
+            if count == lq:
+                return False
+    return True
+
+class Layer:
+    def __init__(self, mem_len: int = 100, max_size: int = 6):
+        self.mem_len = mem_len
+        self.common_strings = []
+        self.previously_seen = []
+        self.max_size = max_size+1
+    def __call__(self, input_arr, training: bool = True):
+        o = []
+        li = len(input_arr)
+        for i in range(li):
+            for y, cs in enumerate(self.common_strings):
+                if (i+cs.shape[0]) <= li and (input_arr[i:i+cs.shape[0]] == cs).all():
+                    o.append(y)
+        if training:
+            cl = 0
+            n = None
+            for i, line in enumerate(self.previously_seen):
+                t = longest_common_substring(input_arr, line)
+                l = len(t)
+                if l > cl and l < self.max_size:
+                    cl = l
+                    n = i
+                    r = t
+            if self.previously_seen != []:
+                if n is not None and len(r) > 1:
+                    self.previously_seen.pop(n)
+                    if not_found_in(r, self.common_strings):
+                        self.common_strings.append(r)
+                self.previously_seen = self.previously_seen[-self.mem_len:]
+            self.previously_seen.append(input_arr)
+        return o
+
+def comparefilter(f1, f2):
+    o = 0
+    hss = 0.5
+    for k in f1:
+        if k in f2 and k in f1:
+            o += np.sum((f2[k] > hss)==(f1[k] > hss))
+    return (o >= len(f1)*hss)
+
+class StrConv:
+    def __init__(self, filters: int, size: int = 4):
+        self.filter_amount = filters
+        self.filters = [{} for _ in range(filters)] # [{43: [3 2 0 3]},]
+        self.bias = np.zeros((self.filter_amount,))
+        self.size = 3
+    def regularize(self):
+        for n, f in enumerate(self.filters):
+            for f2 in self.filters[:n]:
+                if random.randint(0, 100) < 10 and comparefilter(f, f2):
+                    self.filters[n] = {}
+    def __call__(self, input_arr, training: bool = True, debug=False):
+        if len(input_arr) <= self.size:
+            return []
+        o = np.zeros((input_arr.shape[0]-self.size, self.filter_amount))
+        for i in range(input_arr.shape[0]-self.size):
+            for n, c in enumerate(input_arr[i:i+self.size]):
+                for fn, f in enumerate(self.filters):
+                    if c in f:
+                        o[i, fn] += f[c][n]
+        o += self.bias
+        m = np.max(np.abs(o))
+        if m != 0: o /= m
+        if debug:
+            plt.imshow(o)
+            plt.show()
+        if training:
+            for i in range(input_arr.shape[0]-self.size):
+                for n, c in enumerate(input_arr[i:i+self.size]):
+                    for fn, f in enumerate(self.filters):
+                        if c in f:
+#                            s = np.sum(f[c])
+#                            if s > 1000:
+#                                f[c] = (f[c]/(s/(self.size*1000))).astype(np.int64)
+                            self.filters[fn][c][n] = o[i, fn]*0.1+f[c][n]*0.9
+                        else:
+                            f[c] = np.random.uniform(0, 1, (self.size))
+                            f[c][n] = o[i, fn]
+#            for t in range(self.size, input_arr.shape[0]):
+#                for f in range(self.filter_amount):
+#                    self.filters[f] = o[t-self.size, f]
+        """
+        s = 0
+        for a in self.filters:
+            for b in a:
+                s += np.sum(b)
+        if s > 100:
+            s /= self.filter_amount
+            for a in self.filters:
+                for b in a:
+                    a[b] = (a[b]/s).astype(dtype=np.int64)
+        """
+        self.bias -= np.sum(o, axis=0)# / o.shape[0]
+#        print(o)
+        maxed = np.zeros((o.shape[0],)) # could have different outputs, not only max of o, like o>(self.size//2) or o without processing
+        for i in range(maxed.shape[0]):
+            maxed[i] = np.argmax(o[i])
+        return maxed
+
+with open("dataset.txt", "r") as f:
+    lines = f.read().rstrip("\n").split("\n")[:40000]
+
+w = {}
+w2 = {}
+
+c = 0
+
+#layer = Layer(mem_len=1000, max_size=4)
+#layer2 = Layer(mem_len=1000, max_size=6)
+
+with open("l1_large.pckl", "rb") as f: layer = pickle.load(f)
+with open("l2_large.pckl", "rb") as f: layer2 = pickle.load(f)
+with open("w1_large.pckl", "rb") as f: w = pickle.load(f)
+with open("w2_large.pckl", "rb") as f: w2 = pickle.load(f)
+"""
+for n, text in tqdm(enumerate(lines[:-1])):
+    if text.strip() != "" and lines[n+1].strip() != "" and text != lines[n+1]:
+        t = layer(text_to_arr(text), training=True)
+        t = layer(text_to_arr(text), training=False)
+        c += 1
+#        if c == 10:
+#            c = 0
+#            layer.regularize()
+#            layer2.regularize()
+        if len(t) != 0:
+            t2 = layer2(np.array(t), training=True)
+            t2 = layer2(np.array(t), training=False)
+            for a in t2:
+                if a in w2:
+                    w2[a].append(n+1)
+                else:
+                    w2[a] = [n+1,]
+        for a in t:
+            if a in w:
+                w[a].append(n+1)
+            else:
+                w[a] = [n+1,]
+
+for n, text in tqdm(enumerate(lines[:200])):
+    if text.strip() != "" and lines[n+1].strip() != "" and text != lines[n+1]:
+        t = layer(text_to_arr(text), training=True)
+        t = layer(text_to_arr(text), training=False)
+        c += 1
+#        if c == 10:
+#            c = 0
+#            layer.regularize()
+#            layer2.regularize()
+        if len(t) != 0:
+            t2 = layer2(np.array(t), training=True)
+            t2 = layer2(np.array(t), training=False)
+            for a in t2:
+                if a in w2:
+                    w2[a].append(n+1)
+                else:
+                    w2[a] = [n+1,]
+        for a in t:
+            if a in w:
+                w[a].append(n+1)
+            else:
+                w[a] = [n+1,]
+
+with open("l1_large.pckl", "wb") as f: pickle.dump(layer, f)
+with open("l2_large.pckl", "wb") as f: pickle.dump(layer2, f)
+with open("w1_large.pckl", "wb") as f: pickle.dump(w, f)
+with open("w2_large.pckl", "wb") as f: pickle.dump(w2, f)
+"""
+# print(layer.filters)
+
+#for arr in layer.common_strings:
+#    print(''.join([chr(a) for a in arr]))
+
+print(len(lines), "responses available")
+
+import threeletterai
+
+while True:
+    msg = input("Message: ")
+    if len(msg) < 4:
+        print(threeletterai.getresp(msg))
+        continue
+    processed = layer(text_to_arr(msg), training=False)
+    processed = np.array(processed)
+    processed2 = layer2(processed, training=False)
+#    print(processed)
+#    print(processed2)
+    o = np.zeros(len(lines), dtype=np.int16)
+    for a in processed:
+        if a in w:
+            o[w[a]] += 1
+    for a in processed2:
+        if a in w2:
+            o[w2[a]] += 1
+    print(lines[np.argmax(o)], f"       {np.max(o)} sure")
+