Upload 2 files

rnn_implementation.py

@@ -0,0 +1,422 @@
+import torch
+from torch import nn
+import pandas as pd
+import numpy as np
+import os
+import pickle
+from tqdm import tqdm
+os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
+os.environ['CUDA_VISIBLE_DEVICES']='1'
+
+df=pd.read_csv('/home/vivek.trivedi/Reviews.csv',sep=",")
+reviews=df['Text'].to_numpy()
+def mark_sentiment(rating):
+    if(rating<3):
+        return 0 # negative 
+    else:
+        return 1 # positive
+labels=df['Score'].apply(mark_sentiment).to_numpy()
+print(reviews[:2000])
+print(labels[:20])
+
+from string import punctuation
+
+print(punctuation)
+
+all_text = '\n'.join(reviews)
+
+# split by new lines and spaces
+reviews_split = all_text.split('\n')
+all_text = ' '.join(reviews_split)
+
+# create a list of words
+words = all_text.split()
+
+words[:30]
+
+# feel free to use this import
+from collections import Counter
+
+## Build a dictionary that maps words to integers
+counts = Counter(words)
+vocab = sorted(counts, key=counts.get, reverse=True)
+vocab_to_int = {word: ii for ii, word in enumerate(vocab,1)}
+
+## use the dict to tokenize each review in reviews_split
+## store the tokenized reviews in reviews_ints
+reviews_ints = []
+for review in reviews_split:
+  reviews_ints.append([vocab_to_int[word] for word in review.split()])
+
+# stats about vocabulary
+print('Unique words: ', len((vocab_to_int)))  # should ~ 74000+
+print()
+
+# print tokens in first review
+print('Tokenized review: \n', reviews_ints[:1])
+
+encoded_labels = labels
+
+# outlier review stats
+review_lens = Counter([len(x) for x in reviews_ints])
+print("Zero-length reviews: {}".format(review_lens[0]))
+print("Maximum review length: {}".format(max(review_lens)))
+
+print('Number of reviews before removing outliers: ', len(reviews_ints))
+
+## remove any reviews/labels with zero length from the reviews_ints list.
+
+## get any indices of any reviews with length 0
+non_zero_idx = [ii for ii, review in enumerate(reviews_ints) if len(review) != 0]
+
+# remove 0-length review with their labels
+reviews_ints = [reviews_ints[ii] for ii in non_zero_idx]
+encoded_labels = np.array([encoded_labels[ii] for ii in non_zero_idx])
+
+print('Number of reviews after removing outliers: ', len(reviews_ints))
+
+def pad_features(reviews_ints, seq_length):
+    ''' Return features of review_ints, where each review is padded with 0's
+        or truncated to the input seq_length.
+    '''
+    ## getting the correct rows x cols shape
+    features = np.zeros((len(reviews_ints), seq_length), dtype=int)
+
+    ## for each review, I grab that review
+    for i, row in enumerate(reviews_ints):
+      features[i, -len(row):] = np.array(row)[:seq_length]
+
+    return features
+
+# Test your implementation!
+
+seq_length = int(np.mean(list(review_lens.keys())))
+
+features = pad_features(reviews_ints, seq_length=seq_length)
+
+## test statements - do not change - ##
+assert len(features)==len(reviews_ints), "Your features should have as many rows as reviews."
+assert len(features[0])==seq_length, "Each feature row should contain seq_length values."
+
+# print first 10 values of the first 30 batches
+print(features[:30,:10])
+
+split_frac = 0.8
+
+## split data into training, validation, and test data (features and labels, x and y)
+split_idx = int(len(features)*0.8)
+train_x, remaining_x = features[:split_idx], features[split_idx:]
+train_y, remaining_y = encoded_labels[:split_idx], encoded_labels[split_idx:]
+
+test_idx = int(len(remaining_x))
+test_y,val_y = remaining_y[:test_idx], remaining_y[test_idx:]
+test_x,val_x = remaining_x[:test_idx], remaining_x[test_idx:]
+
+
+## print out the shapes of your resultant feature data
+print("\t\t\tFeatures Shapes:")
+print("Train set: \t\t{}".format(train_x.shape),
+      "\nValidation set: \t{}".format(val_x.shape),
+      "\nTest set: \t\t{}".format(test_x.shape))
+
+import torch
+from torch.utils.data import TensorDataset, DataLoader
+
+# create Tensor datasets
+train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))
+valid_data = TensorDataset(torch.from_numpy(val_x), torch.from_numpy(val_y))
+test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))
+
+# dataloaders
+batch_size = 20
+
+# make sure to SHUFFLE your data
+
+_ = torch.manual_seed(100)
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+vocab_size = len(vocab_to_int) + 1 # +1 for zero padding + our word tokens
+output_size = 1
+embedding_dim = 300
+hidden_dim = 256
+n_layers = 2
+n_epoch=10
+
+class MyRNN(nn.Module):
+    def __init__(self, num_layers, hidden_size):
+        super(MyRNN, self).__init__()
+        self.embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.num_layers = num_layers
+        self.hidden_size = hidden_size
+        self.rnn = nn.RNN(
+            input_size=embedding_dim,
+            hidden_size=hidden_size,
+            num_layers=num_layers,
+            batch_first=True
+        )
+        self.fc = nn.Linear(hidden_size,1)
+        self.sig=nn.Sigmoid()
+
+    def forward(self, x):
+        batch_size = x.size(0)
+        embeds = self.embedding(x)
+        hidden_state = self.init_hidden(batch_size).to(device)
+        output, hidden_state = self.rnn(embeds,hidden_state)
+        output = self.fc(hidden_state.squeeze())
+        output=self.sig(output)
+        #output = output.view(batch_size, -1)
+        return output[-1]
+    def init_hidden(self,batch_size):
+        return torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device)
+
+def accuracy_loss(model,dataset,criterion):
+  num_correct = 0
+  num_samples = len(dataset)*batch_size
+  model.eval()
+  loss_=0
+  with torch.no_grad():
+      for name, label in dataset:
+          output = model(name.to(device))
+          loss = criterion(output.float(), label.view(-1,1).to(device).float())
+          pred = torch.round(output.squeeze())
+          num_correct += sum(pred == label.to(device)).cpu().numpy()
+          loss_+=loss.item()
+  return (num_correct / num_samples,loss_/num_samples)
+
+hiden_size_list=[64*i for i in range(1,6)]
+learning_rate_list=[1e-5,1e-4,1e-3,1e-2]
+# accuracy_list={}
+# for learning_rate in tqdm(learning_rate_list):
+#   accuracy_list[learning_rate]={}
+#   for hidden_size in tqdm(hiden_size_list):
+#     model = MyRNN(2, hidden_size).to(device)
+#     criterion = nn.BCELoss()
+#     optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+#     for epoch in range(n_epoch):
+#         acc_epoch=[]
+#         model.train().to(device)
+#         train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
+#         #valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)
+#         test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)
+#         for  name, label in train_loader:
+#             model.zero_grad()
+#             output = model(name.to(device))
+#             loss = criterion(output.float(), label.view(-1,1).to(device).float())
+#             loss.backward()
+#             optimizer.step()
+#         acc_epoch.append([accuracy_loss(model,train_loader,criterion),accuracy_loss(model,test_loader,criterion)])
+#         print('learning rate =',learning_rate,'hidden size =',hidden_size,'epoch =',epoch,'\n train accuracy,train loss,test accuracy,test loss',acc_epoch[-1])
+#     accuracy_list[learning_rate][hidden_size]=acc_epoch
+#     with open("/home/vivek.trivedi/accuracy_loss_list_RNN.pkl",'wb') as file:
+#         pickle.dump(accuracy_list,file)
+
+class MyGRU(nn.Module):
+    def __init__(self, num_layers, hidden_size):
+        super(MyGRU, self).__init__()
+        self.embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.num_layers = num_layers
+        self.hidden_size = hidden_size
+        self.gru = nn.GRU(
+            input_size=embedding_dim,
+            hidden_size=hidden_size,
+            num_layers=num_layers,
+            batch_first=True
+        )
+        self.fc = nn.Linear(hidden_size,1)
+        self.sig=nn.Sigmoid()
+
+    def forward(self, x):
+        batch_size = x.size(0)
+        embeds = self.embedding(x)
+        hidden_state = self.init_hidden(batch_size).to(device)
+        output, hidden_state = self.gru(embeds,hidden_state)
+        output = self.fc(hidden_state.squeeze())
+        output=self.sig(output)
+        #output = output.view(batch_size, -1)
+        return output[-1]
+    def init_hidden(self,batch_size):
+        return torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device)
+
+hiden_size_list=[64*i for i in range(1,6)]
+learning_rate_list=[1e-5,1e-4,1e-3,1e-2]
+# accuracy_list={}
+# for learning_rate in tqdm(learning_rate_list):
+#   accuracy_list[learning_rate]={}
+#   for hidden_size in tqdm(hiden_size_list):
+#     model = MyGRU(2, hidden_size).to(device)
+#     criterion = nn.BCELoss()
+#     optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
+#     acc_epoch=[]
+#     for epoch in range(n_epoch):
+#         model.train().to(device)
+#         train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
+#         #valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)
+#         test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)
+#         for  name, label in tqdm(train_loader):
+#             model.zero_grad()
+#             output = model(name.to(device))
+#             loss = criterion(output.float(), label.view(-1,1).to(device).float())
+#             loss.backward()
+#             optimizer.step()
+#         acc_epoch.append([accuracy_loss(model,train_loader,criterion),accuracy_loss(model,test_loader,criterion)])
+#         print('learning rate =',learning_rate,'hidden size =',hidden_size,'epoch =',epoch,'\n train accuracy,train loss,test accuracy,test loss',acc_epoch[-1])
+#     accuracy_list[learning_rate][hidden_size]=acc_epoch
+#     with open("/home/vivek.trivedi/accuracy_loss_list_gru.pkl",'wb') as file:
+#       pickle.dump(accuracy_list,file)
+
+import torch.nn as nn
+
+class SentimentRNN(nn.Module):
+    """
+    The RNN model that will be used to perform Sentiment analysis.
+    """
+
+    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):
+        """
+        Initialize the model by setting up the layers.
+        """
+        super(SentimentRNN, self).__init__()
+
+        self.output_size = output_size
+        self.n_layers = n_layers
+        self.hidden_dim = hidden_dim
+
+        # embedding and LSTM layers
+        self.embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers,
+                            dropout=drop_prob, batch_first=True)
+
+        # dropout layer
+        self.dropout = nn.Dropout(0.3)
+
+        # linear and sigmoid layer
+        self.fc = nn.Linear(hidden_dim, output_size)
+        self.sig = nn.Sigmoid()
+
+    def forward(self, x):
+        """
+        Perform a forward pass of our model on some input and hidden state.
+        """
+       
+        batch_size = x.size(0)
+        hidden = self.init_hidden(batch_size)
+        # embeddings and lstm_out
+        embeds = self.embedding(x)
+        lstm_out, hidden = self.lstm(embeds, hidden)
+
+        # stack up lstm outputs
+        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)
+
+        # dropout and fully connected layer
+        out = self.dropout(lstm_out)
+        out = self.fc(out)
+
+        # sigmoid function
+        sig_out = self.sig(out)
+
+        # reshape to be batch_size first
+        sig_out = sig_out.view(batch_size, -1)
+        sig_out = sig_out[:, -1] # get last batch of labels
+
+        # return last sigmoid output and hidden state
+        return sig_out, hidden
+
+
+    def init_hidden(self, batch_size):
+        ''' Initializes hidden state '''
+        # Create two new tensors with sizes n_layers x batch_size x hidden_dim,
+        # initialized to zero, for hidden state and cell state of LSTM
+        weight = next(self.parameters()).data
+
+        hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().to(device),
+                   weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().to(device))
+
+        return hidden
+
+def accuracy_loss(net,loader):
+    losses = [] # track loss
+    num_correct = 0
+
+    # init hidden state
+
+
+    net.eval()
+    # iterate over test data
+    for inputs, labels in loader:
+
+        # Creating new variables for the hidden state, otherwise
+        # we'd backprop through the entire training history
+
+        inputs, labels = inputs.to(device), labels.to(device)
+
+        # get predicted outputs
+        output, h = net(inputs)
+
+        # calculate loss
+        loss = criterion(output.squeeze(), labels.float())
+        losses.append(loss.item())
+
+        # convert output probabilities to predicted class (0 or 1)
+        pred = torch.round(output.squeeze())  # rounds to the nearest integer
+
+        # compare predictions to true label
+        correct_tensor = pred.eq(labels.float().view_as(pred))
+        correct = np.squeeze(correct_tensor.cpu().numpy())
+        num_correct += np.sum(correct)
+
+
+    np.mean(losses)
+    acc = num_correct/len(loader.dataset)
+    return acc,np.mean(losses)
+
+# Instantiate the model w/ hyperparams
+vocab_size = len(vocab_to_int) + 1 # +1 for zero padding + our word tokens
+output_size = 1
+embedding_dim = 400
+n_layers = 2
+accuracy_list={}
+for lr in learning_rate_list:
+    accuracy_list[lr]={}
+    for hidden_dim in hiden_size_list:
+        net = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers).to(device)
+        criterion = nn.BCELoss()
+        optimizer = torch.optim.Adam(net.parameters(), lr=lr)
+
+        counter = 0
+        print_every = 100
+        clip=5 # gradient clipping
+        acc_epoch=[]
+        for e in range(n_epoch):
+            train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
+            #valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)
+            test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)
+            # initialize hidden state
+            
+            # batch loop
+            net.train()
+            for inputs, labels in tqdm(train_loader):
+                counter += 1
+
+                inputs, labels = inputs.to(device), labels.to(device)
+
+                # Creating new variables for the hidden state, otherwise
+                # we'd backprop through the entire training history
+
+                # zero accumulated gradients
+                net.zero_grad()
+
+                # get the output from the model
+                output, h = net(inputs)
+
+                # calculate the loss and perform backprop
+                loss = criterion(output.squeeze(), labels.float())
+                loss.backward()
+                # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
+                nn.utils.clip_grad_norm_(net.parameters(), clip)
+                optimizer.step()
+            acc_epoch.append([accuracy_loss(net,train_loader),accuracy_loss(net,test_loader)])
+            print('learning rate =',lr,'hidden size =',hidden_dim,'epoch =',e,'\n train accuracy,train loss,test accuracy,test loss',acc_epoch[-1])
+        accuracy_list[lr][hidden_dim]=acc_epoch
+        with open("/home/vivek.trivedi/accuracy_loss_list_lstm.pkl",'wb') as file:
+            pickle.dump(accuracy_list,file)
+