Upload model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+config.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26df1c5bae0d1013f5211eb1ce268622d68d8dfccdd186759695ad355dd2c473
+size 402241252

config_gzipembed.py

@@ -0,0 +1,28 @@
+from transformers import PretrainedConfig
+from nltk.corpus import stopwords
+from typing import List
+import nltk
+nltk.download('stopwords')
+nltk.download('punkt')
+
+class GZIPEmbeddingConfig(PretrainedConfig):
+    model_type = "gzipembed"
+    def __init__(
+        self,
+        normalize = True,
+        normalized_corpus = True,
+        reduction = False,
+        reduced_dimension = 0,
+        remove_stop_words = True,
+        stop_words = stopwords.words('english'),
+        corpus = [],
+        **kwargs,
+    ):
+        self.corpus = corpus
+        self.normalize = normalize
+        self.normalized_corpus = normalized_corpus 
+        self.reduction = reduction 
+        self.reduced_dimension = reduced_dimension,
+        self.remove_stop_words = remove_stop_words
+        self.stop_words = stop_words
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:753257f36aa4df7a8fa009cef5293a24b945e66727c9831cafedcf24bfd077ae
+size 116

modeling_gzipembed.py

@@ -0,0 +1,53 @@
+from transformers import PreTrainedModel
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from .config_gzipembed import *
+from tqdm.auto import tqdm
+import torch
+import gzip
+
+class GZIPEmbeddingModel(PreTrainedModel):
+    config_class = GZIPEmbeddingConfig
+    def __init__(self, config):
+        super().__init__(config)
+        if config.reduction:
+            self.reduction_head = torch.nn.Linear(len(config.corpus), config.reduced_dimension)
+        else:
+            self.reduction_head = None
+            self.dummy_parameter = torch.nn.Parameter(torch.ones(1))
+
+    def forward(self, texts, verbose=False, return_tensor=False):
+        x = [self.gzip_embed(self.config.corpus, text, verbose=verbose) for text in texts]
+        if self.reduction_head is not None:
+            x = torch.tensor(x)
+            x = x.to(self.reduction_head.dtype).to(self.reduction_head.device)
+            return self.reduction_head(x)
+        return x if not return_tensor else torch.tensor(x)
+
+    def normalize(self, x):
+        x = ''.join([char for char in x.lower() if char in "abcdefghijklmnopqrstuvwxyz "])
+        x = word_tokenize(x)
+        x = [w for w in x if not w in self.config.stop_words]
+        return ' '.join(x)
+
+    def ncd(self, x, y):
+        _x = self.normalize(x) if self.config.normalize else x
+        _y = self.normalize(y) if (not self.config.normalized_corpus) and self.config.normalize else y
+        x_c = len(gzip.compress(_x.encode()))
+        y_c = len(gzip.compress(_y.encode()))
+        xy_c = len(gzip.compress(f"{_x} {_y}".encode()))
+        return (xy_c-min(x_c,y_c))/max(x_c,y_c) 
+    
+    def gzip_embed(
+        self,
+        corpus, 
+        document, 
+        verbose=False,
+    ):
+        embedding = []
+        for reference_document in (corpus if not verbose else tqdm(corpus)):
+            embedding.append(self.ncd(reference_document, document))
+        return embedding
+    
+    def dimensionality(self):
+        return len(self.config.corpus)