add emoji preprocessor

functions.py

@@ -1,14 +1,25 @@
 from transformers import AutoTokenizer, AutoConfig, AutoModelForSequenceClassification
 from scipy.special import softmax
-import emoji
+
 # Define the preprocess function
-def preprocess(text):
+def process_text(text):
     new_text = []
     for t in text.split(" "):
-        t = '@user' if t.startswith('@') and len(t) > 1 else t
-        t = 'http' if t.startswith('http') else t
-        t = emoji.demojize(t)  # Convert emojis to text representation
-        new_text.append(t)
+        # Check if the token is an emoji
+        if emoji.is_emoji(t):
+            # Add the emoji to the new text list
+            new_text.append(t)
+        else:
+            # Check if the token starts with '@' and has more than one character
+            if t.startswith('@') and len(t) > 1:
+                # Replace the token with '@user'
+                t = '@user'
+            # Check if the token starts with 'http'
+            if t.startswith('http'):
+                # Replace the token with 'http'
+                t = 'http'
+            # Add the processed token to the new text list
+            new_text.append(t)
     return " ".join(new_text)
 
 # Define the sentiment_analysis function