Update DataPPwithspecial.py

DataPPwithspecial.py

@@ -1,55 +1,55 @@
-import pandas as pd
-import numpy as np
-from sklearn.model_selection import train_test_split
-import tensorflow as tf
-
-def preprocess():
-    # Load dataset
-    data = pd.read_csv('./backend/English_To_Klingon.csv')
-
-
-    # Append <BOS> and <EOS> tags to the Klingon sentences
-    data['klingon'] = data['klingon'].apply(lambda x: '<BOS> ' + x + ' <EOS>')
-
-    # Separate the sentences
-    english_sentences = data['english'].values
-    klingon_sentences = data['klingon'].values
-
-    # Split data into training and testing sets. An 80 - 20 split is used here
-    english_train, english_test, klingon_train, klingon_test = train_test_split(
-        english_sentences, klingon_sentences, test_size=0.2, random_state=42)
-
-    # Initialize tokenizers with specified vocabulary size
-    english_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=5000, oov_token='<UNK>')
-    klingon_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=5000, oov_token='<UNK>')
-
-    # Fit tokenizers on training data
-    english_tokenizer.fit_on_texts(english_train)
-    klingon_tokenizer.fit_on_texts(klingon_train)
-
-    # Tokenize the sentences
-    english_train_sequences = english_tokenizer.texts_to_sequences(english_train)
-    klingon_train_sequences = klingon_tokenizer.texts_to_sequences(klingon_train)
-    english_test_sequences = english_tokenizer.texts_to_sequences(english_test)
-    klingon_test_sequences = klingon_tokenizer.texts_to_sequences(klingon_test)
-
-    # Padding sequences to a fixed length
-    english_train_padded = tf.keras.preprocessing.sequence.pad_sequences(english_train_sequences, maxlen=50, padding='post')
-    klingon_train_padded = tf.keras.preprocessing.sequence.pad_sequences(klingon_train_sequences, maxlen=50, padding='post')
-    english_test_padded = tf.keras.preprocessing.sequence.pad_sequences(english_test_sequences, maxlen=50, padding='post')
-    klingon_test_padded = tf.keras.preprocessing.sequence.pad_sequences(klingon_test_sequences, maxlen=50, padding='post')
-
-    # Prepare target data for training
-    klingon_train_input = klingon_train_padded[:, :-1] # The decoder input, which is the Klingon sentence shifted by one position to the right for training data.
-    klingon_train_target = klingon_train_padded[:, 1:] # The target output, which is the same sentence shifted by one position to the left for training data.
-    klingon_train_target = np.expand_dims(klingon_train_target, -1)
-
-    # Prepare target data for testing
-    klingon_test_input = klingon_test_padded[:, :-1] # The decoder input for testing data.
-    klingon_test_target = klingon_test_padded[:, 1:] # The target output for testing data.
-    klingon_test_target = np.expand_dims(klingon_test_target, -1)
-
-    return (english_tokenizer, klingon_tokenizer, 50, # max_length
-            english_train_padded, klingon_train_input, klingon_train_target,
-            english_test_padded, klingon_test_input, klingon_test_target)
-
+import pandas as pd
+import numpy as np
+from sklearn.model_selection import train_test_split
+import tensorflow as tf
+
+def preprocess():
+    # Load dataset
+    data = pd.read_csv('English_To_Klingon.csv')
+
+
+    # Append <BOS> and <EOS> tags to the Klingon sentences
+    data['klingon'] = data['klingon'].apply(lambda x: '<BOS> ' + x + ' <EOS>')
+
+    # Separate the sentences
+    english_sentences = data['english'].values
+    klingon_sentences = data['klingon'].values
+
+    # Split data into training and testing sets. An 80 - 20 split is used here
+    english_train, english_test, klingon_train, klingon_test = train_test_split(
+        english_sentences, klingon_sentences, test_size=0.2, random_state=42)
+
+    # Initialize tokenizers with specified vocabulary size
+    english_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=5000, oov_token='<UNK>')
+    klingon_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=5000, oov_token='<UNK>')
+
+    # Fit tokenizers on training data
+    english_tokenizer.fit_on_texts(english_train)
+    klingon_tokenizer.fit_on_texts(klingon_train)
+
+    # Tokenize the sentences
+    english_train_sequences = english_tokenizer.texts_to_sequences(english_train)
+    klingon_train_sequences = klingon_tokenizer.texts_to_sequences(klingon_train)
+    english_test_sequences = english_tokenizer.texts_to_sequences(english_test)
+    klingon_test_sequences = klingon_tokenizer.texts_to_sequences(klingon_test)
+
+    # Padding sequences to a fixed length
+    english_train_padded = tf.keras.preprocessing.sequence.pad_sequences(english_train_sequences, maxlen=50, padding='post')
+    klingon_train_padded = tf.keras.preprocessing.sequence.pad_sequences(klingon_train_sequences, maxlen=50, padding='post')
+    english_test_padded = tf.keras.preprocessing.sequence.pad_sequences(english_test_sequences, maxlen=50, padding='post')
+    klingon_test_padded = tf.keras.preprocessing.sequence.pad_sequences(klingon_test_sequences, maxlen=50, padding='post')
+
+    # Prepare target data for training
+    klingon_train_input = klingon_train_padded[:, :-1] # The decoder input, which is the Klingon sentence shifted by one position to the right for training data.
+    klingon_train_target = klingon_train_padded[:, 1:] # The target output, which is the same sentence shifted by one position to the left for training data.
+    klingon_train_target = np.expand_dims(klingon_train_target, -1)
+
+    # Prepare target data for testing
+    klingon_test_input = klingon_test_padded[:, :-1] # The decoder input for testing data.
+    klingon_test_target = klingon_test_padded[:, 1:] # The target output for testing data.
+    klingon_test_target = np.expand_dims(klingon_test_target, -1)
+
+    return (english_tokenizer, klingon_tokenizer, 50, # max_length
+            english_train_padded, klingon_train_input, klingon_train_target,
+            english_test_padded, klingon_test_input, klingon_test_target)
+