| from transformers import PretrainedConfig |
| from nltk.corpus import stopwords |
| from typing import List |
| import nltk |
| nltk.download('stopwords') |
| nltk.download('punkt') |
|
|
| class GZIPEmbeddingConfig(PretrainedConfig): |
| model_type = "gzipembed" |
| def __init__( |
| self, |
| normalize = True, |
| normalized_corpus = True, |
| reduction = False, |
| reduced_dimension = 0, |
| remove_stop_words = True, |
| stop_words = stopwords.words('english'), |
| corpus = [], |
| **kwargs, |
| ): |
| self.corpus = corpus |
| self.normalize = normalize |
| self.normalized_corpus = normalized_corpus |
| self.reduction = reduction |
| self.reduced_dimension = reduced_dimension, |
| self.remove_stop_words = remove_stop_words |
| self.stop_words = stop_words |
| super().__init__(**kwargs) |
|
|
