|
|
from src.modules.tokenizers import * |
|
|
from src.modules.embeddings import * |
|
|
from src.utils.mapper import configmapper |
|
|
|
|
|
|
|
|
class Preprocessor: |
|
|
def preprocess(self): |
|
|
pass |
|
|
|
|
|
|
|
|
@configmapper.map("preprocessors", "glove") |
|
|
class GlovePreprocessor(Preprocessor): |
|
|
"""GlovePreprocessor.""" |
|
|
|
|
|
def __init__(self, config): |
|
|
""" |
|
|
Args: |
|
|
config (src.utils.module.Config): configuration for preprocessor |
|
|
""" |
|
|
super(GlovePreprocessor, self).__init__() |
|
|
self.config = config |
|
|
self.tokenizer = configmapper.get_object( |
|
|
"tokenizers", self.config.main.preprocessor.tokenizer.name |
|
|
)(**self.config.main.preprocessor.tokenizer.init_params.as_dict()) |
|
|
self.tokenizer_params = ( |
|
|
self.config.main.preprocessor.tokenizer.init_vector_params.as_dict() |
|
|
) |
|
|
|
|
|
self.tokenizer.initialize_vectors(**self.tokenizer_params) |
|
|
self.embeddings = configmapper.get_object( |
|
|
"embeddings", self.config.main.preprocessor.embedding.name |
|
|
)( |
|
|
self.tokenizer.text_field.vocab.vectors, |
|
|
self.tokenizer.text_field.vocab.stoi[self.tokenizer.text_field.pad_token], |
|
|
) |
|
|
|
|
|
def preprocess(self, model_config, data_config): |
|
|
train_dataset = configmapper.get_object("datasets", data_config.main.name)( |
|
|
data_config.train, self.tokenizer |
|
|
) |
|
|
val_dataset = configmapper.get_object("datasets", data_config.main.name)( |
|
|
data_config.val, self.tokenizer |
|
|
) |
|
|
model = configmapper.get_object("models", model_config.name)( |
|
|
self.embeddings, **model_config.params.as_dict() |
|
|
) |
|
|
|
|
|
return model, train_dataset, val_dataset |
|
|
|
|
|
|
|
|
@configmapper.map("preprocessors", "clozePreprocessor") |
|
|
class ClozePreprocessor(Preprocessor): |
|
|
"""GlovePreprocessor.""" |
|
|
|
|
|
def __init__(self, config): |
|
|
""" |
|
|
Args: |
|
|
config (src.utils.module.Config): configuration for preprocessor |
|
|
""" |
|
|
super(ClozePreprocessor, self).__init__() |
|
|
self.config = config |
|
|
self.tokenizer = configmapper.get_object( |
|
|
"tokenizers", self.config.main.preprocessor.tokenizer.name |
|
|
).from_pretrained( |
|
|
**self.config.main.preprocessor.tokenizer.init_params.as_dict() |
|
|
) |
|
|
|
|
|
def preprocess(self, model_config, data_config): |
|
|
train_dataset = configmapper.get_object("datasets", data_config.main.name)( |
|
|
data_config.train, self.tokenizer |
|
|
) |
|
|
val_dataset = configmapper.get_object("datasets", data_config.main.name)( |
|
|
data_config.val, self.tokenizer |
|
|
) |
|
|
model = configmapper.get_object("models", model_config.name).from_pretrained( |
|
|
**model_config.params.as_dict() |
|
|
) |
|
|
|
|
|
return model, train_dataset, val_dataset |
|
|
|
|
|
|
|
|
@configmapper.map("preprocessors", "transformersConcretenessPreprocessor") |
|
|
class TransformersConcretenessPreprocessor(Preprocessor): |
|
|
"""BertConcretenessPreprocessor.""" |
|
|
|
|
|
def __init__(self, config): |
|
|
""" |
|
|
Args: |
|
|
config (src.utils.module.Config): configuration for preprocessor |
|
|
""" |
|
|
super(TransformersConcretenessPreprocessor, self).__init__() |
|
|
self.config = config |
|
|
self.tokenizer = configmapper.get_object( |
|
|
"tokenizers", self.config.main.preprocessor.tokenizer.name |
|
|
).from_pretrained( |
|
|
**self.config.main.preprocessor.tokenizer.init_params.as_dict() |
|
|
) |
|
|
|
|
|
def preprocess(self, model_config, data_config): |
|
|
|
|
|
train_dataset = configmapper.get_object("datasets", data_config.main.name)( |
|
|
data_config.train, self.tokenizer |
|
|
) |
|
|
val_dataset = configmapper.get_object("datasets", data_config.main.name)( |
|
|
data_config.val, self.tokenizer |
|
|
) |
|
|
|
|
|
model = configmapper.get_object("models", model_config.name)( |
|
|
**model_config.params.as_dict() |
|
|
) |
|
|
|
|
|
return model, train_dataset, val_dataset |
|
|
|
