Added some files

preprocessing_test.py

@@ -0,0 +1,866 @@
+from loguru import logger
+import pandas as pd
+import json
+from datetime import datetime
+import ast
+import numpy as np
+from pymongo import MongoClient
+from collections import defaultdict
+
+from tqdm import tqdm
+import time
+
+import requests
+import json
+import os
+import pandas as pd
+import nltk
+from nltk.tokenize import sent_tokenize, word_tokenize
+from nltk.corpus import stopwords
+from textblob import TextBlob
+import re
+from transformers import BertTokenizer, BertModel
+from transformers import RobertaTokenizer, RobertaModel
+import torch
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+# Download NLTK resources
+nltk.download('punkt')
+nltk.download('averaged_perceptron_tagger')
+nltk.download('stopwords')
+nltk.download('punkt_tab')
+nltk.download('averaged_perceptron_tagger_eng')
+
+class Preprocessor:
+    def __init__(self,df):
+        self.df=df
+        self.tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+        self.model = RobertaModel.from_pretrained('roberta-base')
+        self.stop_words = set(stopwords.words('english'))
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  # Add this line
+
+
+
+    def get_bert_embedding(self, text):
+        inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512)
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        return outputs.last_hidden_state.mean(dim=1).squeeze().numpy()
+    
+    def preprocess_text(self,text):
+        return text if pd.notna(text) else ""
+
+    
+    def calculate_duration(self, time_range):
+        if not isinstance(time_range, str) or "-" not in time_range:
+            return None
+        start_str, end_str = time_range.split('-')
+        start_str = start_str.strip() + ':00' if len(start_str.split(':')) == 1 else start_str.strip()
+        end_str = end_str.strip() + ':00' if len(end_str.split(':')) == 1 else end_str.strip()
+        try:
+            start = datetime.strptime(start_str, '%H:%M')
+            end = datetime.strptime(end_str, '%H:%M')
+            duration = (end - start).total_seconds() / 3600  
+            return duration if duration >= 0 else duration + 24  
+        except ValueError:
+            return None
+    def calculate_sentiment_severity(self, text):
+        if pd.isna(text) or not text.strip():
+            return pd.Series({"good_severity": 0.0, "bad_severity": 0.0})
+        
+        # Get sentiment polarity (-1 to 1)
+        blob = TextBlob(text)
+        polarity = blob.sentiment.polarity
+        
+        # Define severity weights
+        good_weight = 0.7
+        bad_weight = 0.3
+        
+        if polarity > 0:  
+            good_severity = good_weight * polarity  
+            bad_severity = 0.0
+        elif polarity < 0:  
+            good_severity = 0.0
+            bad_severity = bad_weight * abs(polarity)  
+        else:  # Neutral (polarity = 0)
+            good_severity = 0.0
+            bad_severity = 0.0
+        
+        return pd.Series({"good_severity": good_severity, "bad_severity": bad_severity})
+
+    
+    def get_avg_duration(self, hours_str):
+        if pd.isna(hours_str) or not isinstance(hours_str, str):
+            return pd.NA  
+        try:
+            hours_dict = ast.literal_eval(hours_str)
+            if not hours_dict:  
+                return pd.NA
+            durations = [self.calculate_duration(time_range) for time_range in hours_dict.values()]
+            valid_durations = [d for d in durations if d is not None]
+            return sum(valid_durations) / len(valid_durations) if valid_durations else pd.NA
+        except (ValueError, SyntaxError, ZeroDivisionError):
+            return pd.NA
+
+
+    def calculate_time_since_last_review(self):
+        present_date = datetime.now()
+        user_latest_timestamp = {}
+    
+        # Convert review_date to datetime
+        self.df["review_date"] = pd.to_datetime(self.df["review_date"])
+    
+        # Calculate hours difference for each user's latest review
+        for user_id in self.df["user_id"].unique():
+            latest_date = self.df[self.df["user_id"] == user_id]["review_date"].max()
+            
+            if not isinstance(latest_date, datetime):
+                latest_date = latest_date.to_pydatetime()
+            
+            hours_difference = (present_date - latest_date).total_seconds() / 3600
+            user_latest_timestamp[user_id] = hours_difference
+    
+        # Map the hours difference to a new column
+        self.df["time_since_last_review_user"] = self.df["user_id"].map(user_latest_timestamp)
+
+    def calculate_time_since_last_review_business(self):
+        present_date = datetime.now()
+        
+        # Ensure review_date is in datetime format
+        self.df["review_date"] = pd.to_datetime(self.df["review_date"])
+    
+        # Initialize dictionary to store hours since last review for each business
+        business_latest_timestamp = {}
+    
+        # Iterate over unique business_ids
+        for business_id in self.df["business_id"].unique():
+            # Get the latest review date for this business
+            latest_date = self.df[self.df["business_id"] == business_id]["review_date"].max()
+            
+            # Convert to datetime object if needed
+            if not isinstance(latest_date, datetime):
+                latest_date = latest_date.to_pydatetime()
+            
+            # Calculate hours difference (already in hours)
+            hours_difference = (present_date - latest_date).total_seconds() / 3600
+            business_latest_timestamp[business_id] = hours_difference
+    
+        # Map the hours difference to the new column
+        self.df["time_since_last_review_business"] = self.df["business_id"].map(business_latest_timestamp)
+
+
+
+    def calculate_user_account_age(self):
+        present_date = datetime.now()
+    
+        # Convert yelping_since to datetime
+        self.df["yelping_since"] = pd.to_datetime(self.df["yelping_since"])
+    
+        # Calculate user account age in days
+        self.df["user_account_age"] = (present_date - self.df["yelping_since"]).dt.days
+
+
+    def calculate_avg_time_between_reviews(self):
+        # Ensure review_date is in datetime format
+        self.df["review_date"] = pd.to_datetime(self.df["review_date"])
+    
+        # Sort the DataFrame by user_id and review_date to ensure chronological order
+        self.df = self.df.sort_values(["user_id", "review_date"])
+    
+        # Define helper function to calculate average time between reviews
+        def calculate_avg_time(group):
+            if len(group) == 1:
+                return 0  # If only one review, assign 0
+            # Calculate differences in hours between consecutive reviews
+            diffs = group["review_date"].diff().dt.total_seconds() / 3600
+            # Drop the first NaN (from diff) and compute the mean
+            return diffs.dropna().mean()
+    
+        # Apply the function to each user_id group and create a mapping
+        avg_time_per_user = self.df.groupby("user_id").apply(calculate_avg_time)
+    
+        # Map the average time back to the original DataFrame
+        self.df["average_time_between_reviews"] = self.df["user_id"].map(avg_time_per_user)
+
+
+    def calculate_user_degree(self):
+    # Calculate the number of unique businesses per user
+        user_business_counts = self.df.groupby("user_id")["business_id"].nunique()
+    
+        # Map the counts back to the original DataFrame
+        self.df["user_degree"] = self.df["user_id"].map(user_business_counts)
+    
+    
+    def calculate_business_degree(self):
+        # Calculate the number of unique users per business
+        business_user_counts = self.df.groupby("business_id")["user_id"].nunique()
+    
+        # Map the counts back to the original DataFrame
+        self.df["business_degree"] = self.df["business_id"].map(business_user_counts)
+    
+    
+    def calculate_rating_variance_user(self):
+        # Calculate the mode (most frequent rating) per user
+        user_rating_mode = self.df.groupby("user_id")["review_stars"].agg(lambda x: x.mode()[0])
+    
+        # Map the most frequent rating back to the original DataFrame
+        self.df["rating_variance_user"] = self.df["user_id"].map(user_rating_mode)
+
+
+    def calculate_user_review_burst_count(self):
+    # Ensure review_date is in datetime format
+        self.df["review_date"] = pd.to_datetime(self.df["review_date"])
+    
+        # Sort by user_id and review_date for chronological order
+        self.df = self.df.sort_values(["user_id", "review_date"])
+    
+        # Function to calculate the max number of reviews in any 20-day window
+        def calculate_burst_count(group):
+            if len(group) <= 1:
+                return 0  # No burst if 1 or fewer reviews
+            
+            # Convert review_date to a Series for rolling window
+            dates = group["review_date"]
+            
+            # Calculate the number of reviews within 20 days of each review
+            burst_counts = []
+            for i, date in enumerate(dates):
+                # Count reviews within 20 days after this date
+                window_end = date + pd.Timedelta(days=20)
+                count = ((dates >= date) & (dates <= window_end)).sum()
+                burst_counts.append(count)
+            
+            # Return the maximum burst count for this user
+            return max(burst_counts)
+    
+        # Calculate the burst count per user
+        user_burst_counts = self.df.groupby("user_id").apply(calculate_burst_count)
+    
+        # Map the burst count back to the original DataFrame
+        self.df["user_review_burst_count"] = self.df["user_id"].map(user_burst_counts)
+    
+    
+    def calculate_business_review_burst_count(self):
+        # Ensure review_date is in datetime format
+        self.df["review_date"] = pd.to_datetime(self.df["review_date"])
+    
+        # Sort by business_id and review_date for chronological order
+        self.df = self.df.sort_values(["business_id", "review_date"])
+    
+        # Function to calculate the max number of reviews in any 10-day window
+        def calculate_burst_count(group):
+            if len(group) <= 1:
+                return 0  # No burst if 1 or fewer reviews
+            
+            # Convert review_date to a Series for rolling window
+            dates = group["review_date"]
+            
+            # Calculate the number of reviews within 10 days of each review
+            burst_counts = []
+            for i, date in enumerate(dates):
+                # Count reviews within 10 days after this date
+                window_end = date + pd.Timedelta(days=10)
+                count = ((dates >= date) & (dates <= window_end)).sum()
+                burst_counts.append(count)
+            
+            # Return the maximum burst count for this business
+            return max(burst_counts)
+    
+        # Calculate the burst count per business
+        business_burst_counts = self.df.groupby("business_id").apply(calculate_burst_count)
+    
+        # Map the burst count back to the original DataFrame
+        self.df["business_review_burst_count"] = self.df["business_id"].map(business_burst_counts)
+        
+    
+    def calculate_temporal_similarity(self):
+        self.df["review_date"] = pd.to_datetime(self.df["review_date"])
+    
+        # Extract the day of the week (0 = Monday, 6 = Sunday)
+        self.df["day_of_week"] = self.df["review_date"].dt.dayofweek
+    
+        # Function to calculate avg hours between reviews on frequent days
+        def calculate_avg_hours_on_frequent_days(group):
+            frequent_days = group["day_of_week"].mode().tolist()
+            
+            if len(group) <= 1:
+                return 0
+            
+            frequent_reviews = group[group["day_of_week"].isin(frequent_days)]
+            
+            if len(frequent_reviews) <= 1:
+                return 0
+            
+            frequent_reviews = frequent_reviews.sort_values("review_date")
+            diffs = frequent_reviews["review_date"].diff().dt.total_seconds() / 3600
+            
+            return diffs.dropna().mean()
+    
+        # Calculate average hours for each user
+        avg_hours_per_user = self.df.groupby("user_id").apply(calculate_avg_hours_on_frequent_days)
+    
+        # Map the average hours to the new column
+        self.df["temporal_similarity"] = self.df["user_id"].map(avg_hours_per_user)
+    
+        # Drop temporary column
+        self.df = self.df.drop(columns=["day_of_week"])
+
+
+    def calculate_rating_deviation_from_business_average(self):
+    # Calculate the average rating per business
+        business_avg_rating = self.df.groupby("business_id")["review_stars"].mean()
+    
+        # Map the average rating to each row
+        self.df["business_avg_rating"] = self.df["business_id"].map(business_avg_rating)
+    
+        # Calculate the deviation from the business average
+        self.df["rating_deviation_from_business_average"] = (
+            self.df["review_stars"] - self.df["business_avg_rating"]
+        )
+    
+        # Drop the temporary column
+        self.df = self.df.drop(columns=["business_avg_rating"])
+    
+    def calculate_review_like_ratio(self):
+        # Create a binary column for liked reviews (stars >= 4)
+        self.df["is_liked"] = (self.df["review_stars"] >= 4).astype(int)
+    
+        # Calculate the like ratio per user
+        user_like_ratio = self.df.groupby("user_id")["is_liked"].mean()
+    
+        # Map the like ratio back to the DataFrame
+        self.df["review_like_ratio"] = self.df["user_id"].map(user_like_ratio)
+    
+        # Drop the temporary column
+        self.df = self.df.drop(columns=["is_liked"])
+    
+    def calculate_latest_checkin_hours(self):
+        self.df["yelping_since"] = pd.to_datetime(self.df["yelping_since"])
+    
+        # Function to get the latest check-in date from a list of strings
+        def get_latest_checkin(checkin_list):
+            if not checkin_list or pd.isna(checkin_list):  # Handle empty or NaN
+                return None
+            if isinstance(checkin_list, str):
+                checkin_dates = checkin_list.split(", ")
+            else:
+                checkin_dates = checkin_list
+            return pd.to_datetime(checkin_dates).max()
+    
+        # Apply the function to get the latest check-in date per row
+        self.df["latest_checkin_date"] = self.df["checkin_date"].apply(get_latest_checkin)
+    
+        # Calculate the hours difference between latest check-in and yelping_since
+        self.df["latest_checkin_hours"] = (
+            (self.df["latest_checkin_date"] - self.df["yelping_since"])
+            .dt.total_seconds() / 3600
+        )
+    
+        # Drop the temporary column
+        self.df = self.df.drop(columns=["latest_checkin_date"])
+        self.df["latest_checkin_hours"].fillna(0,inplace=True)
+        
+    
+    def compute_pronoun_density(self, text):
+        text = self.preprocess_text(text)
+        if not text:
+            return 0
+        words = word_tokenize(text.lower())
+        pos_tags = nltk.pos_tag(words)
+        pronouns = sum(1 for word, pos in pos_tags if pos in ['PRP', 'PRP$'] and word in ['i', 'we'])
+        return pronouns / len(words) if words else 0
+    
+    def compute_avg_sentence_length(self, text):
+        text = self.preprocess_text(text)
+        if not text:
+            return 0
+        sentences = sent_tokenize(text)
+        return sum(len(word_tokenize(sent)) for sent in sentences) / len(sentences) if sentences else 0
+    
+    def compute_excessive_punctuation(self, text):
+        text = self.preprocess_text(text)
+        return len(re.findall(r'[!?.]{2,}', text))
+    
+    def compute_sentiment_polarity(self, text):
+        text = self.preprocess_text(text)
+        return TextBlob(text).sentiment.polarity if text else 0
+
+    def compute_code_switching_flag(self, text):
+        text = self.preprocess_text(text)
+        if not text:
+            return 0
+        
+        tokens = self.tokenizer.tokenize(text.lower())
+        if not tokens:
+            return 0
+        
+        english_words = self.stop_words  # Use self.stop_words from __init__
+        token_set = set(tokens)
+        english_count = sum(1 for token in tokens if token in english_words)
+        
+        non_english_pattern = re.compile(r'[^\x00-\x7F]')
+        has_non_ascii = 1 if non_english_pattern.search(text) else 0
+        
+        english_ratio = english_count / len(tokens) if tokens else 0
+        
+        non_english_tokens = sum(1 for token in token_set if token not in english_words and "##" in token and has_non_ascii)
+        
+        # Flag as code-switching if:
+        # 1. Mixed English presence (ratio between 0.1 and 0.9)
+        # 2. Non-ASCII characters present OR some non-English subword tokens
+        if 0.1 < english_ratio < 0.9 and (has_non_ascii or non_english_tokens > 0):
+            return 1
+        return 0
+    
+    
+    def batch_tokenize(self, texts, batch_size=32, max_length=512):
+        tokenized_outputs = []
+        for i in tqdm(range(0, len(texts), batch_size), desc="Tokenizing with RoBERTa on GPU"):
+            batch_texts = texts[i:i + batch_size]
+            valid_texts = [self.preprocess_text(t) for t in batch_texts]
+            # Tokenize with fixed max_length to ensure consistent tensor sizes
+            inputs = self.tokenizer(valid_texts, return_tensors='pt', truncation=True, padding='max_length', max_length=max_length)
+            tokenized_outputs.append(inputs['input_ids'].to(self.device))  # Move to GPU
+        # Concatenate on GPU with consistent sizes
+        return torch.cat(tokenized_outputs, dim=0)
+    
+    def compute_grammar_error_score(self, texts, tokenized_ids):
+        print("Computing grammar error scores...")
+        error_scores = np.zeros(len(texts), dtype=float)
+        
+        vocab_set = set(self.tokenizer.get_vocab().keys())
+        for i, input_ids in enumerate(tqdm(tokenized_ids, desc="Processing Grammar Errors")):
+            if input_ids.sum() == 0:  # Empty input
+                continue
+            tokens = self.tokenizer.convert_ids_to_tokens(input_ids.cpu().tolist(), skip_special_tokens=True)
+            unknown_count = sum(1 for token in tokens if token not in vocab_set and token not in self.stop_words)
+            total_count = len([t for t in tokens if t not in self.stop_words])
+            error_scores[i] = unknown_count / total_count if total_count > 0 else 0
+        
+        return error_scores
+    
+    def compute_repetitive_words_count(self, texts, tokenized_ids):
+        print("Computing repetitive words counts...")
+        rep_counts = np.zeros(len(texts), dtype=int)
+        
+        for i, input_ids in enumerate(tqdm(tokenized_ids, desc="Processing Repetition")):
+            if input_ids.sum() == 0:  # Empty input
+                continue
+            tokens = self.tokenizer.convert_ids_to_tokens(input_ids.cpu().tolist(), skip_special_tokens=True)
+            valid_tokens = [t for t in tokens if t not in self.stop_words and len(t) > 2]
+            if valid_tokens:
+                token_counts = {}
+                for token in valid_tokens:
+                    token_counts[token] = token_counts.get(token, 0) + 1
+                rep_counts[i] = sum(1 for count in token_counts.values() if count > 1)
+        
+        return rep_counts
+        
+    def preprocess_text_for_similarity(self, text):
+        if pd.isna(text) or not text.strip():
+            return []
+        return [w for w in word_tokenize(str(text).lower()) if w not in self.stop_words]
+    
+    def batch_encode_words(self, texts, batch_size=32, max_length=512):
+        word_lists = [self.preprocess_text_for_similarity(t) for t in tqdm(texts, desc="Tokenizing Texts")]
+        vocab = {word: idx + 1 for idx, word in enumerate(set.union(*[set(w) for w in word_lists if w]))}
+        
+        encoded_batches = []
+        for i in tqdm(range(0, len(word_lists), batch_size), desc="Encoding Words on GPU"):
+            batch_words = word_lists[i:i + batch_size]
+            encoded = np.zeros((len(batch_words), max_length), dtype=np.int64)
+            for j, words in enumerate(batch_words):
+                if words:
+                    word_ids = [vocab.get(w, 0) for w in words][:max_length]
+                    encoded[j, :len(word_ids)] = word_ids
+            encoded_tensor = torch.tensor(encoded, dtype=torch.int64).to(self.device)
+            encoded_batches.append(encoded_tensor)
+        
+        return torch.cat(encoded_batches, dim=0), vocab
+    
+    def compute_similarity_to_other_reviews(self, batch_size=32, max_length=512):
+        all_texts = self.df["review_text"].tolist()
+        all_users = self.df["user_id"].tolist()
+        all_review_ids = self.df["review_id"].tolist()
+        
+        encoded_words, vocab = self.batch_encode_words(all_texts, batch_size, max_length)
+        
+        similarity_scores = {rid: 0.0 for rid in all_review_ids}  # Default scores
+        for i, (review_id, user_id) in enumerate(tqdm(zip(all_review_ids, all_users), desc="Computing Similarities on GPU")):
+            if pd.isna(review_id) or pd.isna(user_id):
+                continue
+            
+            current_words = encoded_words[i]
+            if current_words.sum() == 0:
+                continue
+            
+            other_indices = torch.tensor([j for j, u in enumerate(all_users) if u != user_id and pd.notna(u)], 
+                                       dtype=torch.long).to(self.device)
+            if not other_indices.numel():
+                continue
+            
+            other_words = encoded_words[other_indices]
+            current_set = torch.unique(current_words[current_words > 0])
+            other_flat = other_words[other_words > 0]
+            
+            if other_flat.numel() == 0:
+                continue
+            
+            other_set = torch.unique(other_flat)
+            intersection = torch.sum(torch.isin(current_set, other_set)).float()
+            union = torch.unique(torch.cat([current_set, other_set])).numel()
+            similarity = intersection / union if union > 0 else 0.0
+            
+            similarity_scores[review_id] = similarity.item()
+        return pd.Series(similarity_scores, index=all_review_ids)  
+        
+    def calculate_friend_count(self):
+        friends = []
+        for v in self.df["friends"]:
+            if isinstance(v, str):
+                friends.append(len(v.split(",")))
+            elif type(v)==int or type(v)==float:
+                friends.append(0)
+        self.df["friends"] = friends
+
+    def count_elite_years(self, elite):
+        if pd.isna(elite):  
+            return 0
+        return len(str(elite).split(","))  
+    
+    def transform_elite_status(self):
+        self.df["elite"] = self.df["elite"].apply(lambda x: True if self.count_elite_years(x) > 1 else False)
+        self.df["elite"] = self.df["elite"].astype(int)
+
+
+    def calculate_review_useful_funny_cool(self):
+        self.df["review_useful"] = pd.to_numeric(self.df["review_useful"], errors='coerce').fillna(0)
+        self.df["review_funny"] = pd.to_numeric(self.df["review_funny"], errors='coerce').fillna(0)
+        self.df["review_cool"] = pd.to_numeric(self.df["review_cool"], errors='coerce').fillna(0)
+        self.df["review_useful_funny_cool"] = (
+            self.df["review_useful"] + 
+            self.df["review_funny"] + 
+            self.df["review_cool"]
+        )
+        self.df["review_useful_funny_cool"] = self.df["review_useful_funny_cool"].fillna(0).astype(int)
+        
+    
+    def calculate_user_useful_funny_cool(self):
+        self.df["user_useful_funny_cool"] = (
+            self.df["user_useful"] + 
+            self.df["user_funny"] + 
+            self.df["user_cool"]
+        )
+        self.df["user_useful_funny_cool"] = self.df["user_useful_funny_cool"].fillna(0).astype(int)
+    
+    def compute_fake_score(self, row):
+        suspicion_points = 0
+        
+        # Linguistic Features
+        if row["pronoun_density"] < 0.01:  # Low personal engagement
+            suspicion_points += 1
+        if row["avg_sentence_length"] < 5 or row["avg_sentence_length"] > 30:  # Extreme lengths
+            suspicion_points += 1
+        if row["grammar_error_score"] > 5:  # Many errors
+            suspicion_points += 1
+        if row["repetitive_words_count"] > 5:  # High repetition
+            suspicion_points += 1
+        if row["code_switching_flag"] == 1:  # Language mixing
+            suspicion_points += 1
+        if row["excessive_punctuation_count"] > 3:  # Overuse of punctuation
+            suspicion_points += 1
+        if abs(row["sentiment_polarity"]) > 0.8:  # Extreme sentiment
+            suspicion_points += 1
+        
+        # Review Patterns
+        if row["similarity_to_other_reviews"] > 0.8:  # High duplication
+            suspicion_points += 1
+        if row["user_review_burst_count"] > 5:  # Spammy bursts
+            suspicion_points += 1
+        if row["business_review_burst_count"] > 5:  # Targeted bursts
+            suspicion_points += 1
+        if abs(row["rating_deviation_from_business_average"]) > 2:  # Large rating deviation
+            suspicion_points += 1
+        if row["review_like_ratio"] > 0.9 or row["review_like_ratio"] < 0.1:  # Extreme like ratio
+            suspicion_points += 1
+        
+        # User Behavior
+        if row["user_account_age"] < 30:  # Very new account (days)
+            suspicion_points += 1
+        if row["average_time_between_reviews"] < 24:  # Rapid reviews (hours)
+            suspicion_points += 1
+        if row["user_degree"] < 2:  # Low business interaction
+            suspicion_points += 1
+        if row["time_since_last_review_user"] < 24:  # Recent burst (hours)
+            suspicion_points += 1
+        
+        # Threshold: 3 or more points = fake
+        return 1 if suspicion_points >= 3 else 0
+
+    def dropping_unncessary_columns(self):
+       
+        self.df.drop("review_text", axis=1, inplace=True)
+        self.df.drop("review_date", axis=1, inplace=True)
+        self.df.drop("business_name", axis=1, inplace=True)
+        self.df.drop("address", axis=1, inplace=True)
+        self.df.drop("city", axis=1, inplace=True)
+        self.df.drop("state", axis=1, inplace=True)
+        self.df.drop("postal_code", axis=1, inplace=True)
+        self.df.drop("categories", axis=1, inplace=True)
+        self.df.drop("user_name", axis=1, inplace=True)
+        self.df.drop("yelping_since", axis=1, inplace=True)
+        self.df.drop("checkin_date", axis=1, inplace=True)
+        self.df.drop("review_useful", axis=1, inplace=True)
+        self.df.drop("review_funny", axis=1, inplace=True)
+        self.df.drop("review_cool", axis=1, inplace=True)
+        self.df.drop("user_useful", axis=1, inplace=True)
+        self.df.drop("user_funny", axis=1, inplace=True)
+        self.df.drop("user_cool", axis=1, inplace=True)
+        self.df.drop("is_open", axis=1, inplace=True)
+        self.df.drop("compliment_hot", axis=1, inplace=True)
+        self.df.drop("compliment_more", axis=1, inplace=True)
+        self.df.drop("compliment_profile", axis=1, inplace=True)
+        self.df.drop("compliment_cute", axis=1, inplace=True)
+        self.df.drop("compliment_list", axis=1, inplace=True)
+        self.df.drop("compliment_note", axis=1, inplace=True)
+        self.df.drop("compliment_plain", axis=1, inplace=True)
+        self.df.drop("compliment_cool", axis=1, inplace=True)
+        self.df.drop("compliment_funny", axis=1, inplace=True)
+        self.df.drop("compliment_writer", axis=1, inplace=True)
+        self.df.drop("compliment_photos", axis=1, inplace=True)
+       
+    def run_pipeline(self):
+
+        
+        
+        logger.info("FINALYZING HOURS COLUMN ...")
+        self.df["hours"] = self.df["hours"].apply(self.get_avg_duration)
+        self.df["hours"] = self.df["hours"].fillna(0)
+        print(self.df["hours"][:10])
+        print(self.df["hours"].isnull().sum())
+
+
+        
+
+        logger.info("FINALYZING ATTRIBUTES COLUMN ...")
+        self.df.drop("attributes",axis=1,inplace=True)
+
+
+        
+        logger.info("CREATING time_since_last_review_user COLUMN ...")
+        self.calculate_time_since_last_review()
+        print(np.unique(self.df["time_since_last_review_user"] ))
+
+        
+        logger.info("CREATING time_since_last_review_business COLUMN ...")
+        self.calculate_time_since_last_review_business()
+        print(np.unique(self.df["time_since_last_review_business"] ))
+
+
+
+        logger.info("CREATING user_account_age COLUMN ...")
+        self.calculate_user_account_age()
+        print(np.unique(self.df["user_account_age"] ))
+
+
+
+        logger.info("CREATING average_time_between_reviews COLUMN ...")
+        self.calculate_avg_time_between_reviews()
+        print(np.unique(self.df["average_time_between_reviews"] ))
+
+
+
+        logger.info("CREATING user_degree COLUMN ...")
+        self.calculate_user_degree()
+        print(np.unique(self.df["user_degree"] ))
+
+        
+        logger.info("CREATING business_degree COLUMN ...")
+        self.calculate_business_degree()
+        print(np.unique(self.df["business_degree"] ))
+
+
+        logger.info("CREATING rating_variance_user COLUMN ...")
+        self.calculate_rating_variance_user()
+        print(np.unique(self.df["rating_variance_user"] ))
+
+
+
+        logger.info("CREATING user_review_burst_count COLUMN ...")
+        self.calculate_user_review_burst_count()
+        print(np.unique(self.df["user_review_burst_count"] ))
+
+
+        logger.info("CREATING business_review_burst_count COLUMN ...")
+        self.calculate_business_review_burst_count()
+        print(np.unique(self.df["business_review_burst_count"] ))
+
+
+        
+        logger.info("CREATING temporal_similarity COLUMN ...")
+        self.calculate_temporal_similarity()
+        print(np.unique(self.df["temporal_similarity"] ))
+
+
+
+        logger.info("CREATING rating_deviation_from_business_average COLUMN ...")
+        self.calculate_rating_deviation_from_business_average()
+        print(np.unique(self.df["rating_deviation_from_business_average"] ))
+
+
+
+        logger.info("CREATING review_like_ratio COLUMN ...")
+        self.calculate_review_like_ratio()
+        print(np.unique(self.df["review_like_ratio"] ))
+
+
+
+        logger.info("CREATING latest_checkin_hours COLUMN ...")
+        self.calculate_latest_checkin_hours()
+        print(np.unique(self.df["latest_checkin_hours"] ))
+
+
+
+
+        logger.info("CREATING pronoun_density COLUMN ...")
+        self.df["pronoun_density"] = self.df["review_text"].apply(self.compute_pronoun_density)
+        print(np.unique(self.df["pronoun_density"] ))
+
+        logger.info("CREATING avg_sentence_length COLUMN ...")
+        self.df["avg_sentence_length"] = self.df["review_text"].apply(self.compute_avg_sentence_length)
+        print(np.unique(self.df["avg_sentence_length"] ))
+
+        logger.info("CREATING excessive_punctuation_count COLUMN ...")
+        self.df["excessive_punctuation_count"] = self.df["review_text"].apply(self.compute_excessive_punctuation)
+        print(np.unique(self.df["excessive_punctuation_count"] ))
+
+        logger.info("CREATING sentiment_polarity COLUMN ...")
+        self.df["sentiment_polarity"] = self.df["review_text"].apply(self.compute_sentiment_polarity)
+        print(np.unique(self.df["sentiment_polarity"] ))
+
+        logger.info("CREATING good_severity and  bad_severity COLUMNS ...")
+        severity_scores = self.df["review_text"].apply(self.calculate_sentiment_severity)
+        self.df[["good_severity", "bad_severity"]] = severity_scores
+        print(np.unique(self.df["good_severity"] ))
+        print(np.unique(self.df["bad_severity"] ))
+
+
+        logger.info("CREATING code_switching_flag COLUMN ...")
+        self.df["code_switching_flag"] = self.df["review_text"].apply(self.compute_code_switching_flag)
+        print(np.unique(self.df["code_switching_flag"] ))
+
+
+        all_texts = self.df["review_text"].tolist()
+        tokenized_ids = self.batch_tokenize(all_texts, batch_size=32, max_length=512)
+            
+        logger.info("CREATING grammar_error_score COLUMN ...")
+        self.df["grammar_error_score"] = self.compute_grammar_error_score(all_texts, tokenized_ids)
+        print(np.unique(self.df["grammar_error_score"] ))
+
+
+        logger.info("CREATING repetitive_words_count COLUMN ...")
+        self.df["repetitive_words_count"] = self.compute_repetitive_words_count(all_texts, tokenized_ids)
+        print(np.unique(self.df["repetitive_words_count"] ))
+
+
+
+        logger.info("CREATING similarity_to_other_reviews COLUMN ...")
+        similarity_scores = self.compute_similarity_to_other_reviews(batch_size=32, max_length=512)
+        self.df["similarity_to_other_reviews"] = self.df["review_id"].map(similarity_scores)
+        
+        print(np.unique(self.df["similarity_to_other_reviews"] ))
+
+
+
+        logger.info("CREATING friends COLUMN ...")    
+        self.calculate_friend_count()
+        print(self.df["friends"].value_counts())
+
+        logger.info("CREATING elite COLUMN ...")
+        self.transform_elite_status()
+        print(self.df["elite"].value_counts())
+
+
+        logger.info("CREATING review_useful_funny_cool COLUMN ...")
+        self.calculate_review_useful_funny_cool()
+        print(self.df["review_useful_funny_cool"].value_counts())
+
+
+        logger.info("CREATING user_useful_funny_cool COLUMN ...")
+        self.calculate_user_useful_funny_cool()
+        print(self.df["user_useful_funny_cool"].value_counts())
+
+
+        # logger.info("CREATING LABEL COLUMN ...")
+        # self.df["fake"] = self.df.apply(self.compute_fake_score, axis=1)
+        # print(self.df["fake"].value_counts())
+        
+        logger.info("DELETING THE UNWANTED COLUMNS ...")
+        self.dropping_unncessary_columns()
+        print()
+
+        logger.info("SEEING NULL VALUES IN FINAL COLUMNS.....")
+        print(set(self.df.isnull().sum().values))
+        
+
+        
+
+        return self.df
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        
+        
+        
+        
+
+
+        
+
+
+
+      
+
+
+
+
+
+
+
+        
+
+
+
+
+
+
+
+
+        
+
+
+
+
+
+
+        
+
+    
+        
+        
+            
+
+        
+