Updated trainer

src/pipes/const.py

@@ -1,3 +1,5 @@
 data_dir: str = "E:/bn_multi_tribe_mt/data/"
 langs: list[str] = ['bn', 'en', 'gr']
 MAX_SEQ_LEN = 30
+BATCH_SIZE = 64
+BUFFER_SIZE = 10000

src/pipes/data.py

@@ -3,6 +3,7 @@ import const
 import utils
 import string
 
+
 class SequenceLoader:
     def __init__(self):
         self.sequence_dict = None
@@ -38,6 +39,12 @@ class SequenceLoader:
         self.lang = lang
 
 
+def serialize(src_seq, tar_seq):
+    tar_seq_in = tar_seq[:, :-1].to_tensor()
+    tar_seq_out = tar_seq[:, 1:].to_tensor()
+    return (src_seq, tar_seq_in), tar_seq_out
+
+
 def remove_punctuation_from_seq(seq):
     english_punctuations = string.punctuation
     bangla_punctuations = "৷-–—’‘৳…।"
@@ -157,6 +164,29 @@ class Dataset:
             seq_processor.pad()
         self.dataset_dict = seq_processor.get_dict()
 
+    def pull(self):
+        src_lang_train_seqs = self.dataset_dict[self.langs[0]]["train"]
+        tar_lang_train_seqs = self.dataset_dict[self.langs[1]]["train"]
+
+        src_lang_val_seqs = self.dataset_dict[self.langs[0]]["val"]
+        tar_lang_val_seqs = self.dataset_dict[self.langs[1]]["val"]
+
+        train_ds = ((tf.data.Dataset
+                     .from_tensor_slices((src_lang_train_seqs, tar_lang_train_seqs)))
+                    .shuffle(const.BUFFER_SIZE)
+                    .batch(const.BATCH_SIZE))
+
+        val_ds = (tf.data.Dataset
+                  .from_tensor_slices(src_lang_val_seqs, tar_lang_val_seqs)
+                  .shuffle(const.BUFFER_SIZE)
+                  .batch(const.BATCH_SIZE))
+
+        train_ds = train_ds.map(serialize, tf.data.AUTOTUNE)
+        val_ds = val_ds.map(serialize, tf.data.AUTOTUNE)
+
+        return trainset, valset
+
+    @staticmethod
     def get_dict(self):
         return self.dataset_dict
 
@@ -167,4 +197,6 @@ if __name__ == "__main__":
     dataset_dict = dataset_object.get_dict()
     utils.save_dict("{}/dataset.txt".format(const.data_dir), dataset_dict)
     dataset_object.process()
+    trainset, valset = dataset_object.pull()
+
     print(utils.load_dict("{}/dataset.txt".format(const.data_dir)))

src/pipes/models.py

@@ -34,38 +34,6 @@ class Seq2Seq:
         outputs = self.output_layer(decoder_outputs)
         self.model = tf.keras.Model([encoder_inputs, decoder_inputs], outputs)
 
-    def run(self, encoder_input_data, decoder_input_data, val_encoder_input_data, val_decoder_input_data):
-        self.model.compile(
-            optimizer=self.optimizer,
-            loss=self.loss,
-            metrics=self.metrics
-        )
-
-        decoder_target_data = [[sentence[1:] + [0]] for sentence in decoder_input_data]
-        val_decoder_target_data = [[sentence[1:] + [0]] for sentence in val_decoder_input_data]
-
-        self.model.fit(
-            ([encoder_input_data, decoder_input_data]),
-            decoder_target_data,
-            batch_size=self.batch_size,
-            epochs=self.epochs,
-            validation_data=([val_encoder_input_data, val_decoder_input_data], val_decoder_target_data)
-        )
-
     def get(self):
         return self.model
 
-    def set_epochs(self, epochs):
-        self.epochs = epochs
-
-    def set_batch_size(self, batch_size):
-        self.batch_size = batch_size
-
-    def set_loss(self, loss):
-        self.loss = loss
-
-    def set_optimizer(self, optimizer):
-        self.optimizer = optimizer
-
-    def set_metric(self, metrics):
-        self.metrics = metrics