Update app.py

app.py

@@ -1,250 +1,42 @@
-import pandas as pd
-import numpy as np
-import json
-import ast
-from sklearn.model_selection import train_test_split
-from sklearn.linear_model import LinearRegression, LogisticRegression
-from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
-from xgboost import XGBRegressor, XGBClassifier
-from sklearn.svm import SVC
-from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
-from statsmodels.tsa.arima.model import ARIMA
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.layers import LSTM, Dense
-from tensorflow.keras.callbacks import EarlyStopping
-from sklearn.preprocessing import MinMaxScaler, LabelEncoder
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-from textblob import TextBlob  # For sentiment analysis
-from imblearn.over_sampling import SMOTE  # For handling imbalanced data
-import logging
-import matplotlib.pyplot as plt
-from statsmodels.tsa.stattools import adfuller  # For stationarity check
+# Install Prophet if not already installed
+!pip install prophet
 
-# Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+# Import Prophet
+from prophet import Prophet
 
-# Define mean_absolute_percentage_error function
-def mean_absolute_percentage_error(y_true, y_pred):
-    y_true, y_pred = np.array(y_true), np.array(y_pred)
-    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
+# Remove posting_time_encoded if it's not available
+if 'posting_time_encoded' in solved_df.columns:
+    solved_df.drop(columns=['posting_time_encoded'], inplace=True)
 
-# Load engagement_metrics.json
-logging.info("Loading engagement metrics...")
-try:
-    with open('engagement_metrics.json', 'r') as f:
-        engagement_metrics = json.load(f)
-    engagement_df = pd.json_normalize(engagement_metrics)
-except FileNotFoundError:
-    logging.error("engagement_metrics.json not found. Please ensure the file exists.")
-    exit()
+# Time-Series Model: Optimal Posting Times (using Prophet)
+logging.info("Training time-series model for optimal posting times using Prophet...")
+time_series_data = engagement_summary[['posting_time', 'engagement_rate']].rename(columns={'posting_time': 'ds', 'engagement_rate': 'y'})
 
-# Load solved.json (hashtags and captions)
-logging.info("Loading solved.json...")
-try:
-    with open('solved.json', 'r') as f:
-        solved_data = json.load(f)
-    solved_df = pd.json_normalize(solved_data)
-except FileNotFoundError:
-    logging.error("solved.json not found. Please ensure the file exists.")
-    exit()
+# Train Prophet model
+prophet_model = Prophet()
+prophet_model.fit(time_series_data)
 
-# Check for required columns in engagement data
-required_columns = ['posting_time', 'likes', 'comments', 'shares']
-missing_columns = [col for col in required_columns if col not in engagement_df.columns]
+# Make future predictions
+future = prophet_model.make_future_dataframe(periods=30)  # Predict for the next 30 days
+forecast = prophet_model.predict(future)
 
-if missing_columns:
-    logging.warning(f"Missing required columns in engagement_metrics.json: {missing_columns}")
-    for col in missing_columns:
-        engagement_df[col] = 0  # Fill with default value
-    logging.info("Default values added for missing columns.")
-
-# Handle missing values in engagement data
-engagement_df.fillna({
-    'likes': 0,
-    'comments': 0,
-    'shares': 0
-}, inplace=True)
-
-# Calculate engagement_rate
-engagement_df['engagement_rate'] = engagement_df['likes'] + engagement_df['comments'] + engagement_df['shares']
-
-# Convert posting_time to datetime in engagement data
-logging.info("Converting posting_time to datetime...")
-engagement_df['posting_time'] = pd.to_datetime(engagement_df['posting_time'], format='%Y-%m-%d %H:%M:%S', errors='coerce')
-
-# Ensure 'caption' is treated as a string column in solved data
-solved_df['caption'] = solved_df['caption'].astype(str)
-
-# Extract hashtags from the solved data (already provided as a list)
-logging.info("Extracting hashtags from solved data...")
-solved_df['hashtags'] = solved_df['hashtags'].apply(lambda x: x if isinstance(x, list) else [])
-
-# Filter out rows with invalid posting_time in engagement data
-engagement_df = engagement_df[engagement_df['posting_time'].notna()]
-
-# Convert posting_time to Unix timestamp in engagement data (for time-based operations)
-logging.info("Converting posting_time to Unix timestamp...")
-engagement_df['posting_time_encoded'] = engagement_df['posting_time'].astype(int) / 10**9
-
-# Ensure required columns exist in the solved dataset
-if 'content_type' not in solved_df.columns:
-    solved_df['content_type'] = 'photo'  # Default value (adjust based on your data)
-
-if 'media_type' not in solved_df.columns:
-    solved_df['media_type'] = 'image'  # Default value (adjust based on your data)
-
-# Encode categorical columns in the solved dataset
-label_encoder = LabelEncoder()
-solved_df['content_type_encoded'] = label_encoder.fit_transform(solved_df['content_type'])
-solved_df['media_type_encoded'] = label_encoder.fit_transform(solved_df['media_type'])
-
-# Calculate sentiment for captions in the solved dataset
-logging.info("Performing sentiment analysis on captions...")
-solved_df['caption_sentiment'] = solved_df['caption'].apply(lambda x: TextBlob(x).sentiment.polarity)
-
-# Use caption sentiment as the overall sentiment
-solved_df['sentiment'] = solved_df['caption_sentiment']
-
-# Feature Engineering in the solved dataset
-logging.info("Performing feature engineering...")
-solved_df['caption_length'] = solved_df['caption'].apply(len)
-solved_df['hashtag_count'] = solved_df['hashtags'].apply(len)
-
-# Analyze engagement data separately
-logging.info("Analyzing engagement data separately...")
-engagement_summary = engagement_df.groupby('posting_time').agg({
-    'likes': 'sum',
-    'comments': 'sum',
-    'shares': 'sum',
-    'engagement_rate': 'mean'
-}).reset_index()
-
-# Plot engagement rate over time
-plt.figure(figsize=(10, 6))
-plt.plot(engagement_summary['posting_time'], engagement_summary['engagement_rate'])
-plt.title('Engagement Rate Over Time')
-plt.xlabel('Time')
+# Plot the forecast
+fig = prophet_model.plot(forecast)
+plt.title('Engagement Rate Forecast (Prophet)')
+plt.xlabel('Date')
 plt.ylabel('Engagement Rate')
 plt.show()
 
-# Time-Series Model: Optimal Posting Times (using engagement data)
-logging.info("Training time-series model for optimal posting times...")
-time_series_data = engagement_summary.set_index('posting_time')
-
-# Check for NaN values
-print("NaN values in time-series data:")
-print(time_series_data.isnull().sum())
-
-# Check if the 'engagement_rate' column is empty or entirely NaN
-if time_series_data['engagement_rate'].isnull().all() or len(time_series_data) == 0:
-    logging.warning("The 'engagement_rate' column is empty or entirely NaN. Skipping stationarity check and ARIMA modeling.")
-else:
-    # Check stationarity using ADF test
-    engagement_rate_cleaned = time_series_data['engagement_rate'].dropna()
-    if len(engagement_rate_cleaned) == 0:
-        logging.warning("The 'engagement_rate' column is empty after dropping NaN values. Skipping stationarity check.")
-    else:
-        result = adfuller(engagement_rate_cleaned)
-        print('ADF Statistic:', result[0])
-        print('p-value:', result[1])
-        print('Critical Values:', result[4])
-
-        # If the data is not stationary, apply differencing
-        if result[1] > 0.05:
-            print("Data is not stationary. Applying differencing...")
-            time_series_data['engagement_rate_diff'] = time_series_data['engagement_rate'].diff().dropna()
-            time_series_data = time_series_data.dropna()  # Drop NaN values after differencing
-            if len(time_series_data) == 0:
-                logging.warning("The differenced 'engagement_rate' column is empty. Skipping ARIMA modeling.")
-            else:
-                print("ADF test after differencing:")
-                result_diff = adfuller(time_series_data['engagement_rate_diff'])
-                print('ADF Statistic:', result_diff[0])
-                print('p-value:', result_diff[1])
-                print('Critical Values:', result_diff[4])
-
-        # Train ARIMA model on stationary data
-        if len(time_series_data) > 0:
-            train_size = int(len(time_series_data) * 0.8)
-            train, test = time_series_data[:train_size], time_series_data[train_size:]
-            if 'engagement_rate_diff' in time_series_data.columns:
-                arima_model = ARIMA(train['engagement_rate_diff'], order=(5, 1, 0))
-            else:
-                arima_model = ARIMA(train['engagement_rate'], order=(5, 1, 0))
-            arima_fit = arima_model.fit()
-            predictions = arima_fit.forecast(steps=len(test))
-            mape = mean_absolute_percentage_error(test['engagement_rate'], predictions)
-            logging.info(f"ARIMA Model: MAPE: {mape:.4f}")
-
-# Ensure 'hashtags' column is properly formatted
-solved_df['hashtags'] = solved_df['hashtags'].apply(lambda x: x if isinstance(x, list) and len(x) > 0 else ['no_hashtag'])
-
-# Recommendation System: Hashtag and Keyword Recommendations (using solved dataset)
-logging.info("Training recommendation system for hashtags...")
-hashtags = solved_df['hashtags'].apply(lambda x: ' '.join(x))  # Convert list of hashtags to a single string
-
-# Check if hashtags are empty
-if hashtags.str.strip().eq('').all():
-    logging.warning("The 'hashtags' column is empty or contains only stop words. Skipping recommendation system.")
-else:
-    vectorizer = TfidfVectorizer()
-    tfidf_matrix = vectorizer.fit_transform(hashtags)
-    cosine_sim = cosine_similarity(tfidf_matrix, tfidf_matrix)
-
-    def recommend_hashtags(post_index, top_n=5):
-        sim_scores = list(enumerate(cosine_sim[post_index]))
-        sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
-        top_indices = [i[0] for i in sim_scores[1:top_n+1]]
-        return solved_df.iloc[top_indices]['hashtags']
-
-    # Example: Recommend hashtags for the first post
-    logging.info("Example Hashtag Recommendations:")
-    print(recommend_hashtags(0))
-
-# Sentiment Analysis: Audience Reactions (using solved dataset)
-logging.info("Performing sentiment analysis on captions...")
-solved_df['sentiment_category'] = solved_df['sentiment'].apply(lambda x: 'Positive' if x > 0 else 'Negative' if x < 0 else 'Neutral')
-logging.info("Sentiment Analysis Results:")
-print(solved_df['sentiment_category'].value_counts())
-
-# Niche Trend Analysis (using solved dataset)
-logging.info("Analyzing niche trends...")
-niche_trends = solved_df.groupby('content_type')['sentiment'].mean().sort_values(ascending=False)
-logging.info("Top Performing Content Types by Sentiment:")
-print(niche_trends)
-
-# Viral Potential of Posts
-logging.info("Training model for viral potential prediction...")
-viral_threshold = engagement_df['engagement_rate'].quantile(0.9)
-engagement_df['viral'] = engagement_df['engagement_rate'].apply(lambda x: 1 if x >= viral_threshold else 0)
-solved_df['viral'] = engagement_df['viral']
-
-# Features for viral potential prediction
-features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
-X = solved_df[features]
-y = solved_df['viral']
-
-# Split data into training and testing sets
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
-
-# Train a Random Forest Classifier
-viral_model = RandomForestClassifier(random_state=42)
-viral_model.fit(X_train, y_train)
-
 # Evaluate the model
-y_pred = viral_model.predict(X_test)
-accuracy = accuracy_score(y_test, y_pred)
-logging.info(f"Viral Potential Model Accuracy: {accuracy:.4f}")
-
-# Feature importance
-importance = viral_model.feature_importances_
-for feature, score in zip(features, importance):
-    logging.info(f"Feature Importance - {feature}: {score:.4f}")
+from sklearn.metrics import mean_absolute_error
+y_true = time_series_data['y']
+y_pred = forecast.loc[:len(y_true)-1, 'yhat']  # Align predictions with true values
+mae = mean_absolute_error(y_true, y_pred)
+logging.info(f"Prophet Model - MAE: {mae:.4f}")
 
-# Engagement Rate Predictions
+# Engagement Rate Predictions (without posting_time_encoded)
 logging.info("Training model for engagement rate prediction...")
-features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded', 'posting_time_encoded']
+features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
 X = solved_df[features]
 y = engagement_df['engagement_rate']
 
@@ -263,34 +55,4 @@ logging.info(f"Engagement Rate Prediction Model - MAE: {mae:.4f}")
 # Feature importance
 importance = engagement_model.feature_importances_
 for feature, score in zip(features, importance):
-    logging.info(f"Feature Importance - {feature}: {score:.4f}")
-
-# Which Type of Posts Yield Greater Results When Promoted
-logging.info("Training model for promotion prediction...")
-promotion_threshold = engagement_df['engagement_rate'].quantile(0.8)
-engagement_df['promote'] = engagement_df['engagement_rate'].apply(lambda x: 1 if x >= promotion_threshold else 0)
-solved_df['promote'] = engagement_df['promote']
-
-# Features for promotion prediction
-features = ['caption_length', 'hashtag_count', 'sentiment', 'content_type_encoded', 'media_type_encoded']
-X = solved_df[features]
-y = solved_df['promote']
-
-# Split data into training and testing sets
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
-
-# Train a Logistic Regression Model
-promotion_model = LogisticRegression(random_state=42)
-promotion_model.fit(X_train, y_train)
-
-# Evaluate the model
-y_pred = promotion_model.predict(X_test)
-accuracy = accuracy_score(y_test, y_pred)
-logging.info(f"Promotion Prediction Model Accuracy: {accuracy:.4f}")
-
-# Analyze content type impact
-content_type_impact = solved_df.groupby('content_type')['promote'].mean().sort_values(ascending=False)
-logging.info("Content Type Impact on Promotion:")
-print(content_type_impact)
-
-logging.info("Analysis complete!")
+    logging.info(f"Feature Importance - {feature}: {score:.4f}")