model

LSTM/__pycache__/config.cpython-38.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1a316609dde1c8a3aeca1dfe09c8f54ff2bc3193afae97d0ced837735a41063
+size 480

LSTM/__pycache__/inputHandler.cpython-38.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e822d00085a8a0c7f4c9227b1f26fff37af85a5ba9a97005d77ee5628ae0ba4e
+size 6423

LSTM/choosed_checkpoit/lstm_50_50_0.17_0.25.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ca771b21fa23112b534ee0d1f8bacd97cf2205301e7a66bed5c34546201d5c8
+size 40510656

LSTM/config.py

@@ -0,0 +1,25 @@
+
+EMBEDDING_DIM = 50
+
+
+MAX_SEQUENCE_LENGTH = 50
+VALIDATION_SPLIT = 0.1
+
+
+RATE_DROP_LSTM = 0.17
+RATE_DROP_DENSE = 0.25
+NUMBER_LSTM = 50
+NUMBER_DENSE_UNITS = 50
+ACTIVATION_FUNCTION = 'relu'
+
+
+siamese_config = {
+	'EMBEDDING_DIM': EMBEDDING_DIM,
+	'MAX_SEQUENCE_LENGTH' : MAX_SEQUENCE_LENGTH,
+	'VALIDATION_SPLIT': VALIDATION_SPLIT,
+	'RATE_DROP_LSTM': RATE_DROP_LSTM,
+	'RATE_DROP_DENSE': RATE_DROP_DENSE,
+	'NUMBER_LSTM': NUMBER_LSTM,
+	'NUMBER_DENSE_UNITS': NUMBER_DENSE_UNITS,
+	'ACTIVATION_FUNCTION': ACTIVATION_FUNCTION
+}

LSTM/inputHandler.py

@@ -0,0 +1,147 @@
+from keras.preprocessing.sequence import pad_sequences
+from keras.preprocessing.text import Tokenizer
+from gensim.models import Word2Vec
+import numpy as np
+import gc
+
+
+def train_word2vec(documents, embedding_dim):
+    """
+    train word2vector over training documents
+    Args:
+        documents (list): list of document
+        embedding_dim (int): output wordvector size
+    Returns:
+        word_vectors(dict): dict containing words and their respective vectors
+    """
+    model = Word2Vec(documents, min_count=1, size=embedding_dim)
+    word_vectors = model.wv
+    del model
+    return word_vectors
+
+
+def create_embedding_matrix(tokenizer, word_vectors, embedding_dim):
+    """
+    Create embedding matrix containing word indexes and respective vectors from word vectors
+    Args:
+        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object containing word indexes
+        word_vectors (dict): dict containing word and their respective vectors
+        embedding_dim (int): dimension of word vector
+
+    Returns:
+
+    """
+    nb_words = len(tokenizer.word_index) + 1
+    word_index = tokenizer.word_index
+    embedding_matrix = np.zeros((nb_words, embedding_dim))
+    print("Embedding matrix shape: %s" % str(embedding_matrix.shape))
+    for word, i in word_index.items():
+        try:
+            embedding_vector = word_vectors[word]
+            if embedding_vector is not None:
+                embedding_matrix[i] = embedding_vector
+        except KeyError:
+            print("vector not found for word - %s" % word)
+    print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))
+    return embedding_matrix
+
+
+def word_embed_meta_data(documents, embedding_dim):
+    """
+    Load tokenizer object for given vocabs list
+    Args:
+        documents (list): list of document
+        embedding_dim (int): embedding dimension
+    Returns:
+        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object
+        embedding_matrix (dict): dict with word_index and vector mapping
+    """
+    documents = [str(x).lower().split() for x in documents]
+    tokenizer = Tokenizer()
+    tokenizer.fit_on_texts(documents)
+    word_vector = train_word2vec(documents, embedding_dim)
+    embedding_matrix = create_embedding_matrix(tokenizer, word_vector, embedding_dim)
+    del word_vector
+    gc.collect()
+    return tokenizer, embedding_matrix
+
+
+def create_train_dev_set(tokenizer, sentences_pair, is_similar, max_sequence_length, validation_split_ratio):
+    """
+    Create training and validation dataset
+    Args:
+        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object
+        sentences_pair (list): list of tuple of sentences pairs
+        is_similar (list): list containing labels if respective sentences in sentence1 and sentence2
+                           are same or not (1 if same else 0)
+        max_sequence_length (int): max sequence length of sentences to apply padding
+        validation_split_ratio (float): contain ratio to split training data into validation data
+
+    Returns:
+        train_data_1 (list): list of input features for training set from sentences1
+        train_data_2 (list): list of input features for training set from sentences2
+        labels_train (np.array): array containing similarity score for training data
+        leaks_train(np.array): array of training leaks features
+
+        val_data_1 (list): list of input features for validation set from sentences1
+        val_data_2 (list): list of input features for validation set from sentences1
+        labels_val (np.array): array containing similarity score for validation data
+        leaks_val (np.array): array of validation leaks features
+    """
+    sentences1 = [x[0].lower() for x in sentences_pair]
+    sentences2 = [x[1].lower() for x in sentences_pair]
+    train_sequences_1 = tokenizer.texts_to_sequences(sentences1)
+    train_sequences_2 = tokenizer.texts_to_sequences(sentences2)
+    leaks = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))]
+             for x1, x2 in zip(train_sequences_1, train_sequences_2)]
+
+    train_padded_data_1 = pad_sequences(train_sequences_1, maxlen=max_sequence_length)
+    train_padded_data_2 = pad_sequences(train_sequences_2, maxlen=max_sequence_length)
+    train_labels = np.array(is_similar)
+    leaks = np.array(leaks)
+
+    shuffle_indices = np.random.permutation(np.arange(len(train_labels)))
+    train_data_1_shuffled = train_padded_data_1[shuffle_indices]
+    train_data_2_shuffled = train_padded_data_2[shuffle_indices]
+    train_labels_shuffled = train_labels[shuffle_indices]
+    leaks_shuffled = leaks[shuffle_indices]
+
+    dev_idx = max(1, int(len(train_labels_shuffled) * validation_split_ratio))
+
+    del train_padded_data_1
+    del train_padded_data_2
+    gc.collect()
+
+    train_data_1, val_data_1 = train_data_1_shuffled[:-dev_idx], train_data_1_shuffled[-dev_idx:]
+    train_data_2, val_data_2 = train_data_2_shuffled[:-dev_idx], train_data_2_shuffled[-dev_idx:]
+    labels_train, labels_val = train_labels_shuffled[:-dev_idx], train_labels_shuffled[-dev_idx:]
+    leaks_train, leaks_val = leaks_shuffled[:-dev_idx], leaks_shuffled[-dev_idx:]
+
+    return train_data_1, train_data_2, labels_train, leaks_train, val_data_1, val_data_2, labels_val, leaks_val
+
+
+def create_test_data(tokenizer, test_sentences_pair, max_sequence_length):
+    """
+    Create training and validation dataset
+    Args:
+        tokenizer (keras.preprocessing.text.Tokenizer): keras tokenizer object
+        test_sentences_pair (list): list of tuple of sentences pairs
+        max_sequence_length (int): max sequence length of sentences to apply padding
+
+    Returns:
+        test_data_1 (list): list of input features for training set from sentences1
+        test_data_2 (list): list of input features for training set from sentences2
+    """
+    test_sentences1 = [str(x[0]).lower() for x in test_sentences_pair]
+    test_sentences2 = [x[1].lower() for x in test_sentences_pair]
+
+    test_sequences_1 = tokenizer.texts_to_sequences(test_sentences1)
+    test_sequences_2 = tokenizer.texts_to_sequences(test_sentences2)
+    leaks_test = [[len(set(x1)), len(set(x2)), len(set(x1).intersection(x2))]
+                  for x1, x2 in zip(test_sequences_1, test_sequences_2)]
+
+    leaks_test = np.array(leaks_test)
+    test_data_1 = pad_sequences(test_sequences_1, maxlen=max_sequence_length)
+    test_data_2 = pad_sequences(test_sequences_2, maxlen=max_sequence_length)
+
+    return test_data_1, test_data_2, leaks_test

LSTM/sample_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff555eaf20951ddb3f611ef509a5e19d31f6842983df6faab4754a17a6eb854
+size 62661