Update app.py

app.py

@@ -1,12 +1,128 @@
-# Install Prophet if not already installed
-!pip install prophet
+import pandas as pd
+import numpy as np
+import json
+import ast
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LinearRegression, LogisticRegression
+from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
+from xgboost import XGBRegressor, XGBClassifier
+from sklearn.svm import SVC
+from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
+from prophet import Prophet  # For time-series forecasting
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Dense
+from tensorflow.keras.callbacks import EarlyStopping
+from sklearn.preprocessing import MinMaxScaler, LabelEncoder
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from textblob import TextBlob  # For sentiment analysis
+from imblearn.over_sampling import SMOTE  # For handling imbalanced data
+import logging
+import matplotlib.pyplot as plt
+from statsmodels.tsa.stattools import adfuller  # For stationarity check
 
-# Import Prophet
-from prophet import Prophet
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Remove posting_time_encoded if it's not available
-if 'posting_time_encoded' in solved_df.columns:
-    solved_df.drop(columns=['posting_time_encoded'], inplace=True)
+# Define mean_absolute_percentage_error function
+def mean_absolute_percentage_error(y_true, y_pred):
+    y_true, y_pred = np.array(y_true), np.array(y_pred)
+    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+
+# Load engagement_metrics.json
+logging.info("Loading engagement metrics...")
+try:
+    with open('engagement_metrics.json', 'r') as f:
+        engagement_metrics = json.load(f)
+    engagement_df = pd.json_normalize(engagement_metrics)
+except FileNotFoundError:
+    logging.error("engagement_metrics.json not found. Please ensure the file exists.")
+    exit()
+
+# Load solved.json (hashtags and captions)
+logging.info("Loading solved.json...")
+try:
+    with open('solved.json', 'r') as f:
+        solved_data = json.load(f)
+    solved_df = pd.json_normalize(solved_data)
+except FileNotFoundError:
+    logging.error("solved.json not found. Please ensure the file exists.")
+    exit()
+
+# Check for required columns in engagement data
+required_columns = ['posting_time', 'likes', 'comments', 'shares']
+missing_columns = [col for col in required_columns if col not in engagement_df.columns]
+
+if missing_columns:
+    logging.warning(f"Missing required columns in engagement_metrics.json: {missing_columns}")
+    for col in missing_columns:
+        engagement_df[col] = 0  # Fill with default value
+    logging.info("Default values added for missing columns.")
+
+# Handle missing values in engagement data
+engagement_df.fillna({
+    'likes': 0,
+    'comments': 0,
+    'shares': 0
+}, inplace=True)
+
+# Calculate engagement_rate
+engagement_df['engagement_rate'] = engagement_df['likes'] + engagement_df['comments'] + engagement_df['shares']
+
+# Convert posting_time to datetime in engagement data
+logging.info("Converting posting_time to datetime...")
+engagement_df['posting_time'] = pd.to_datetime(engagement_df['posting_time'], format='%Y-%m-%d %H:%M:%S', errors='coerce')
+
+# Ensure 'caption' is treated as a string column in solved data
+solved_df['caption'] = solved_df['caption'].astype(str)
+
+# Extract hashtags from the solved data (already provided as a list)
+logging.info("Extracting hashtags from solved data...")
+solved_df['hashtags'] = solved_df['hashtags'].apply(lambda x: x if isinstance(x, list) else [])
+
+# Filter out rows with invalid posting_time in engagement data
+engagement_df = engagement_df[engagement_df['posting_time'].notna()]
+
+# Ensure required columns exist in the solved dataset
+if 'content_type' not in solved_df.columns:
+    solved_df['content_type'] = 'photo'  # Default value (adjust based on your data)
+
+if 'media_type' not in solved_df.columns:
+    solved_df['media_type'] = 'image'  # Default value (adjust based on your data)
+
+# Encode categorical columns in the solved dataset
+label_encoder = LabelEncoder()
+solved_df['content_type_encoded'] = label_encoder.fit_transform(solved_df['content_type'])
+solved_df['media_type_encoded'] = label_encoder.fit_transform(solved_df['media_type'])
+
+# Calculate sentiment for captions in the solved dataset
+logging.info("Performing sentiment analysis on captions...")
+solved_df['caption_sentiment'] = solved_df['caption'].apply(lambda x: TextBlob(x).sentiment.polarity)
+
+# Use caption sentiment as the overall sentiment
+solved_df['sentiment'] = solved_df['caption_sentiment']
+
+# Feature Engineering in the solved dataset
+logging.info("Performing feature engineering...")
+solved_df['caption_length'] = solved_df['caption'].apply(len)
+solved_df['hashtag_count'] = solved_df['hashtags'].apply(len)
+
+# Analyze engagement data separately
+logging.info("Analyzing engagement data separately...")
+engagement_summary = engagement_df.groupby('posting_time').agg({
+    'likes': 'sum',
+    'comments': 'sum',
+    'shares': 'sum',
+    'engagement_rate': 'mean'
+}).reset_index()
+
+# Plot engagement rate over time
+plt.figure(figsize=(10, 6))
+plt.plot(engagement_summary['posting_time'], engagement_summary['engagement_rate'])
+plt.title('Engagement Rate Over Time')
+plt.xlabel('Time')
+plt.ylabel('Engagement Rate')
+plt.show()
 
 # Time-Series Model: Optimal Posting Times (using Prophet)
 logging.info("Training time-series model for optimal posting times using Prophet...")
@@ -34,7 +150,72 @@ y_pred = forecast.loc[:len(y_true)-1, 'yhat']  # Align predictions with true val
 mae = mean_absolute_error(y_true, y_pred)
 logging.info(f"Prophet Model - MAE: {mae:.4f}")
 
-# Engagement Rate Predictions (without posting_time_encoded)
+# Ensure 'hashtags' column is properly formatted
+solved_df['hashtags'] = solved_df['hashtags'].apply(lambda x: x if isinstance(x, list) and len(x) > 0 else ['no_hashtag'])
+
+# Recommendation System: Hashtag and Keyword Recommendations (using solved dataset)
+logging.info("Training recommendation system for hashtags...")
+hashtags = solved_df['hashtags'].apply(lambda x: ' '.join(x))  # Convert list of hashtags to a single string
+
+# Check if hashtags are empty
+if hashtags.str.strip().eq('').all():
+    logging.warning("The 'hashtags' column is empty or contains only stop words. Skipping recommendation system.")
+else:
+    vectorizer = TfidfVectorizer()
+    tfidf_matrix = vectorizer.fit_transform(hashtags)
+    cosine_sim = cosine_similarity(tfidf_matrix, tfidf_matrix)
+
+    def recommend_hashtags(post_index, top_n=5):
+        sim_scores = list(enumerate(cosine_sim[post_index]))
+        sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
+        top_indices = [i[0] for i in sim_scores[1:top_n+1]]
+        return solved_df.iloc[top_indices]['hashtags']
+
+    # Example: Recommend hashtags for the first post
+    logging.info("Example Hashtag Recommendations:")
+    print(recommend_hashtags(0))
+
+# Sentiment Analysis: Audience Reactions (using solved dataset)
+logging.info("Performing sentiment analysis on captions...")
+solved_df['sentiment_category'] = solved_df['sentiment'].apply(lambda x: 'Positive' if x > 0 else 'Negative' if x < 0 else 'Neutral')
+logging.info("Sentiment Analysis Results:")
+print(solved_df['sentiment_category'].value_counts())
+
+# Niche Trend Analysis (using solved dataset)
+logging.info("Analyzing niche trends...")
+niche_trends = solved_df.groupby('content_type')['sentiment'].mean().sort_values(ascending=False)
+logging.info("Top Performing Content Types by Sentiment:")
+print(niche_trends)
+
+# Viral Potential of Posts
+logging.info("Training model for viral potential prediction...")
+viral_threshold = engagement_df['engagement_rate'].quantile(0.9)
+engagement_df['viral'] = engagement_df['engagement_rate'].apply(lambda x: 1 if x >= viral_threshold else 0)
+solved_df['viral'] = engagement_df['viral']
+
+# Features for viral potential prediction
+features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
+X = solved_df[features]
+y = solved_df['viral']
+
+# Split data into training and testing sets
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# Train a Random Forest Classifier
+viral_model = RandomForestClassifier(random_state=42)
+viral_model.fit(X_train, y_train)
+
+# Evaluate the model
+y_pred = viral_model.predict(X_test)
+accuracy = accuracy_score(y_test, y_pred)
+logging.info(f"Viral Potential Model Accuracy: {accuracy:.4f}")
+
+# Feature importance
+importance = viral_model.feature_importances_
+for feature, score in zip(features, importance):
+    logging.info(f"Feature Importance - {feature}: {score:.4f}")
+
+# Engagement Rate Predictions
 logging.info("Training model for engagement rate prediction...")
 features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
 X = solved_df[features]
@@ -55,4 +236,34 @@ logging.info(f"Engagement Rate Prediction Model - MAE: {mae:.4f}")
 # Feature importance
 importance = engagement_model.feature_importances_
 for feature, score in zip(features, importance):
-    logging.info(f"Feature Importance - {feature}: {score:.4f}")
+    logging.info(f"Feature Importance - {feature}: {score:.4f}")
+
+# Which Type of Posts Yield Greater Results When Promoted
+logging.info("Training model for promotion prediction...")
+promotion_threshold = engagement_df['engagement_rate'].quantile(0.8)
+engagement_df['promote'] = engagement_df['engagement_rate'].apply(lambda x: 1 if x >= promotion_threshold else 0)
+solved_df['promote'] = engagement_df['promote']
+
+# Features for promotion prediction
+features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
+X = solved_df[features]
+y = solved_df['promote']
+
+# Split data into training and testing sets
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# Train a Logistic Regression Model
+promotion_model = LogisticRegression(random_state=42)
+promotion_model.fit(X_train, y_train)
+
+# Evaluate the model
+y_pred = promotion_model.predict(X_test)
+accuracy = accuracy_score(y_test, y_pred)
+logging.info(f"Promotion Prediction Model Accuracy: {accuracy:.4f}")
+
+# Analyze content type impact
+content_type_impact = solved_df.groupby('content_type')['promote'].mean().sort_values(ascending=False)
+logging.info("Content Type Impact on Promotion:")
+print(content_type_impact)
+
+logging.info("Analysis complete!")