update code

.ipynb_checkpoints/tester-checkpoint.ipynb

@@ -0,0 +1,49 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48c76726-b0a4-43e6-9f07-0199e0248d5e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "bd2034e6-1187-4887-9ca7-8b9c0b5c9331",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1ac414f1-37dd-4642-867c-5520a16c1c86",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/utils-checkpoint.py

@@ -37,7 +37,8 @@ def get_single_prediction(text):
     text_vectors = np.mean([w2v[i] for i in text.split()], axis=0)
 
     # Make predictions
-    results = model.predict(text_vectors)
+    results = model.predict(text_vectors.reshape(1,300)).squeeze()
+    print(results)
     
     # Get sentiment
     sentiment = get_sentiment_label_facebook(classifier(text, 
@@ -46,6 +47,10 @@ def get_single_prediction(text):
     
     # Consolidate results
     pred_labels = [labels[idx] for idx, tag in enumerate(results) if tag == 1]
+    
+    if len(pred_labels) == 0:
+        pred_labels.append('others')
+    
     pred_labels.append(sentiment)
     
     return pred_labels
@@ -59,22 +64,27 @@ def get_multiple_predictions(csv):
     df['sequence'] = df['sequence'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation
 
     # Remove OOV words
-    df['sequence'] = df['sequence'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
+    df['sequence_clean'] = df['sequence'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
 
     # Remove rows with blank string
-    invalid = df[(pd.isna(df['sequence'])) | (df['sequence'] == '')]
+    invalid = df[(pd.isna(df['sequence_clean'])) | (df['sequence_clean'] == '')]
+    invalid.drop(columns=['sequence_clean'], inplace=True)
     
+    # Drop rows with blank string
     df.dropna(inplace=True)
-    df = df[df['sequence'] != ''].reset_index(drop=True)
+    df = df[df['sequence_clean'] != ''].reset_index(drop=True)
     
     # Vectorise text and store in new dataframe. Sentence vector = average of word vectors
-    series_text_vectors = pd.DataFrame(df['sequence'].apply(lambda x: np.mean([w2v[i] for i in x.split()], axis=0)).values.tolist())
+    series_text_vectors = pd.DataFrame(df['sequence_clean'].apply(lambda x: np.mean([w2v[i] for i in x.split()], axis=0)).values.tolist())
     
     # Get predictions
     pred_results = pd.DataFrame(model.predict(series_text_vectors), columns = labels)
     
     # Join back to original sequence
-    final_results = df.join(series_text_vectors)
+    final_results = df.join(pred_results)
+    final_results.drop(columns=['sequence_clean'], inplace=True)
+    final_results['others'] = final_results[labels].max(axis=1)
+    final_results['others'] = final_results['others'].apply(lambda x: 1 if x == 0 else 0)
     
     # Get sentiment labels
     final_results['sentiment'] = final_results['sequence'].apply(lambda x: get_sentiment_label_facebook(classifier(x, 

tester.ipynb

@@ -0,0 +1,49 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48c76726-b0a4-43e6-9f07-0199e0248d5e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "bd2034e6-1187-4887-9ca7-8b9c0b5c9331",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1ac414f1-37dd-4642-867c-5520a16c1c86",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

utils.py

@@ -37,7 +37,8 @@ def get_single_prediction(text):
     text_vectors = np.mean([w2v[i] for i in text.split()], axis=0)
 
     # Make predictions
-    results = model.predict(text_vectors)
+    results = model.predict(text_vectors.reshape(1,300)).squeeze()
+    print(results)
     
     # Get sentiment
     sentiment = get_sentiment_label_facebook(classifier(text, 
@@ -46,6 +47,10 @@ def get_single_prediction(text):
     
     # Consolidate results
     pred_labels = [labels[idx] for idx, tag in enumerate(results) if tag == 1]
+    
+    if len(pred_labels) == 0:
+        pred_labels.append('others')
+    
     pred_labels.append(sentiment)
     
     return pred_labels
@@ -59,22 +64,27 @@ def get_multiple_predictions(csv):
     df['sequence'] = df['sequence'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation
 
     # Remove OOV words
-    df['sequence'] = df['sequence'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
+    df['sequence_clean'] = df['sequence'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))
 
     # Remove rows with blank string
-    invalid = df[(pd.isna(df['sequence'])) | (df['sequence'] == '')]
+    invalid = df[(pd.isna(df['sequence_clean'])) | (df['sequence_clean'] == '')]
+    invalid.drop(columns=['sequence_clean'], inplace=True)
     
+    # Drop rows with blank string
     df.dropna(inplace=True)
-    df = df[df['sequence'] != ''].reset_index(drop=True)
+    df = df[df['sequence_clean'] != ''].reset_index(drop=True)
     
     # Vectorise text and store in new dataframe. Sentence vector = average of word vectors
-    series_text_vectors = pd.DataFrame(df['sequence'].apply(lambda x: np.mean([w2v[i] for i in x.split()], axis=0)).values.tolist())
+    series_text_vectors = pd.DataFrame(df['sequence_clean'].apply(lambda x: np.mean([w2v[i] for i in x.split()], axis=0)).values.tolist())
     
     # Get predictions
     pred_results = pd.DataFrame(model.predict(series_text_vectors), columns = labels)
     
     # Join back to original sequence
-    final_results = df.join(series_text_vectors)
+    final_results = df.join(pred_results)
+    final_results.drop(columns=['sequence_clean'], inplace=True)
+    final_results['others'] = final_results[labels].max(axis=1)
+    final_results['others'] = final_results['others'].apply(lambda x: 1 if x == 0 else 0)
     
     # Get sentiment labels
     final_results['sentiment'] = final_results['sequence'].apply(lambda x: get_sentiment_label_facebook(classifier(x,