updated code

.ipynb_checkpoints/utils-checkpoint.py

@@ -57,11 +57,12 @@ def get_multiple_predictions(csv):
     df = pd.read_csv(csv)
     df.columns = ['sequence']
 
-    df['sequence'] = df['sequence'].str.lower() #lower case
-    df['sequence'] = df['sequence'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation
+    df['sequence_clean'] = df['sequence'].str.lower() #lower case
+    df['sequence_clean'] = df['sequence_clean'].str.strip()
+    df['sequence_clean'] = df['sequence_clean'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation
 
     # Remove OOV words
-    df['sequence_clean'] = df['sequence'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
+    df['sequence_clean'] = df['sequence_clean'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
 
     # Remove rows with blank string
     invalid = df[(pd.isna(df['sequence_clean'])) | (df['sequence_clean'] == '')]
@@ -79,16 +80,17 @@ def get_multiple_predictions(csv):
     
     # Join back to original sequence
     final_results = df.join(pred_results)
-    final_results.drop(columns=['sequence_clean'], inplace=True)
     final_results['others'] = final_results[labels].max(axis=1)
     final_results['others'] = final_results['others'].apply(lambda x: 1 if x == 0 else 0)
     
     # Get sentiment labels
-    final_results['sentiment'] = final_results['sequence'].apply(lambda x: get_sentiment_label_facebook(classifier(x, 
+    final_results['sentiment'] = final_results['sequence_clean'].apply(lambda x: get_sentiment_label_facebook(classifier(x, 
                                                             candidate_labels=['positive', 'negative'], 
                                                             hypothesis_template='The sentiment of this is {}'))
                                                                 )
     
+    final_results.drop(columns=['sequence_clean'], inplace=True)
+    
     # Append invalid rows
     if len(invalid) == 0:
         return final_results.to_csv(index=False).encode('utf-8')

utils.py

@@ -57,11 +57,12 @@ def get_multiple_predictions(csv):
     df = pd.read_csv(csv)
     df.columns = ['sequence']
 
-    df['sequence'] = df['sequence'].str.lower() #lower case
-    df['sequence'] = df['sequence'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation
+    df['sequence_clean'] = df['sequence'].str.lower() #lower case
+    df['sequence_clean'] = df['sequence_clean'].str.strip()
+    df['sequence_clean'] = df['sequence_clean'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation
 
     # Remove OOV words
-    df['sequence_clean'] = df['sequence'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
+    df['sequence_clean'] = df['sequence_clean'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
 
     # Remove rows with blank string
     invalid = df[(pd.isna(df['sequence_clean'])) | (df['sequence_clean'] == '')]
@@ -79,16 +80,17 @@ def get_multiple_predictions(csv):
     
     # Join back to original sequence
     final_results = df.join(pred_results)
-    final_results.drop(columns=['sequence_clean'], inplace=True)
     final_results['others'] = final_results[labels].max(axis=1)
     final_results['others'] = final_results['others'].apply(lambda x: 1 if x == 0 else 0)
     
     # Get sentiment labels
-    final_results['sentiment'] = final_results['sequence'].apply(lambda x: get_sentiment_label_facebook(classifier(x, 
+    final_results['sentiment'] = final_results['sequence_clean'].apply(lambda x: get_sentiment_label_facebook(classifier(x, 
                                                             candidate_labels=['positive', 'negative'], 
                                                             hypothesis_template='The sentiment of this is {}'))
                                                                 )
     
+    final_results.drop(columns=['sequence_clean'], inplace=True)
+    
     # Append invalid rows
     if len(invalid) == 0:
         return final_results.to_csv(index=False).encode('utf-8')