Spaces:
Sleeping
Sleeping
Upload full_analysis.py
Browse files- full_analysis.py +461 -0
full_analysis.py
ADDED
|
@@ -0,0 +1,461 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
AIRBNB PRICING & GUEST SATISFACTION OPTIMIZER
|
| 4 |
+
AI for Big Data Management - Group Project
|
| 5 |
+
============================================
|
| 6 |
+
This script performs the full analysis pipeline:
|
| 7 |
+
1. Data loading & cleaning (real-world + synthetic)
|
| 8 |
+
2. Qualitative analysis (VADER sentiment)
|
| 9 |
+
3. Quantitative analysis (Random Forest classification + ARIMA forecasting)
|
| 10 |
+
4. Visualization outputs
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import pandas as pd
|
| 14 |
+
import numpy as np
|
| 15 |
+
import matplotlib
|
| 16 |
+
matplotlib.use('Agg')
|
| 17 |
+
import matplotlib.pyplot as plt
|
| 18 |
+
import seaborn as sns
|
| 19 |
+
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
|
| 20 |
+
from sklearn.ensemble import RandomForestClassifier
|
| 21 |
+
from sklearn.model_selection import train_test_split
|
| 22 |
+
from sklearn.metrics import classification_report, confusion_matrix
|
| 23 |
+
from sklearn.preprocessing import LabelEncoder
|
| 24 |
+
from statsmodels.tsa.arima.model import ARIMA
|
| 25 |
+
import warnings
|
| 26 |
+
warnings.filterwarnings('ignore')
|
| 27 |
+
|
| 28 |
+
np.random.seed(42)
|
| 29 |
+
OUTPUT_DIR = "/content/outputs"
|
| 30 |
+
import os
|
| 31 |
+
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
| 32 |
+
|
| 33 |
+
print("=" * 60)
|
| 34 |
+
print("PHASE 1: DATA GENERATION (Real-world structure + Synthetic)")
|
| 35 |
+
print("=" * 60)
|
| 36 |
+
|
| 37 |
+
# βββ Generate realistic listings data (mirrors Inside Airbnb structure) βββ
|
| 38 |
+
n_listings = 500
|
| 39 |
+
neighbourhoods = ['Le Marais', 'Montmartre', 'Latin Quarter', 'Bastille',
|
| 40 |
+
'Belleville', 'Oberkampf', 'Saint-Germain', 'Pigalle',
|
| 41 |
+
'Batignolles', 'Menilmontant', 'Republique', 'Nation']
|
| 42 |
+
room_types = ['Entire home/apt', 'Private room', 'Shared room']
|
| 43 |
+
room_weights = [0.55, 0.38, 0.07]
|
| 44 |
+
|
| 45 |
+
listings = pd.DataFrame({
|
| 46 |
+
'listing_id': range(1, n_listings + 1),
|
| 47 |
+
'name': [f"Charming {np.random.choice(['Studio','Apt','Loft','Room','Flat'])} in {np.random.choice(neighbourhoods)}" for _ in range(n_listings)],
|
| 48 |
+
'neighbourhood': np.random.choice(neighbourhoods, n_listings),
|
| 49 |
+
'room_type': np.random.choice(room_types, n_listings, p=room_weights),
|
| 50 |
+
'accommodates': np.random.choice([1,2,3,4,5,6], n_listings, p=[0.1,0.3,0.25,0.2,0.1,0.05]),
|
| 51 |
+
'bedrooms': np.random.choice([0,1,2,3], n_listings, p=[0.15,0.5,0.25,0.1]),
|
| 52 |
+
'minimum_nights': np.random.choice([1,2,3,5,7,30], n_listings, p=[0.3,0.25,0.2,0.1,0.1,0.05]),
|
| 53 |
+
'number_of_reviews': np.random.poisson(40, n_listings),
|
| 54 |
+
'reviews_per_month': np.round(np.random.exponential(2.5, n_listings), 2),
|
| 55 |
+
'host_is_superhost': np.random.choice([0, 1], n_listings, p=[0.7, 0.3]),
|
| 56 |
+
'instant_bookable': np.random.choice([0, 1], n_listings, p=[0.4, 0.6]),
|
| 57 |
+
})
|
| 58 |
+
|
| 59 |
+
# Price depends on room type and neighbourhood (realistic)
|
| 60 |
+
base_prices = {'Entire home/apt': 120, 'Private room': 55, 'Shared room': 25}
|
| 61 |
+
premium_neighbourhoods = ['Le Marais', 'Saint-Germain', 'Latin Quarter', 'Montmartre']
|
| 62 |
+
listings['price'] = listings.apply(
|
| 63 |
+
lambda r: base_prices[r['room_type']] * (1.3 if r['neighbourhood'] in premium_neighbourhoods else 1.0)
|
| 64 |
+
* np.random.uniform(0.6, 1.6), axis=1
|
| 65 |
+
).round(2)
|
| 66 |
+
|
| 67 |
+
# Review scores depend on superhost status + noise
|
| 68 |
+
listings['review_scores_rating'] = np.clip(
|
| 69 |
+
np.where(listings['host_is_superhost'] == 1,
|
| 70 |
+
np.random.normal(4.7, 0.2, n_listings),
|
| 71 |
+
np.random.normal(4.3, 0.4, n_listings)),
|
| 72 |
+
3.0, 5.0
|
| 73 |
+
).round(2)
|
| 74 |
+
|
| 75 |
+
print(f"Generated {len(listings)} listings across {len(neighbourhoods)} neighbourhoods")
|
| 76 |
+
print(f"Room type distribution:\n{listings['room_type'].value_counts().to_string()}")
|
| 77 |
+
|
| 78 |
+
# βββ Generate realistic reviews βββ
|
| 79 |
+
review_templates_positive = [
|
| 80 |
+
"Amazing location, very clean and the host was super responsive!",
|
| 81 |
+
"Perfect apartment for our stay. Walking distance to everything.",
|
| 82 |
+
"Loved the cozy atmosphere. Would definitely come back!",
|
| 83 |
+
"Great value for money. The neighborhood is lovely and quiet.",
|
| 84 |
+
"Exceeded expectations! Beautiful decor and comfortable bed.",
|
| 85 |
+
"Host was incredibly helpful with restaurant recommendations.",
|
| 86 |
+
"Spotless apartment with a wonderful view. Highly recommend!",
|
| 87 |
+
"Best Airbnb experience we've had. Smooth check-in process.",
|
| 88 |
+
"Charming place in a fantastic location. Five stars!",
|
| 89 |
+
"Everything was perfect from start to finish. Thank you!",
|
| 90 |
+
"The apartment was exactly as described, very well maintained.",
|
| 91 |
+
"Wonderful stay, the kitchen was fully equipped and very handy.",
|
| 92 |
+
]
|
| 93 |
+
review_templates_neutral = [
|
| 94 |
+
"Decent place, a bit noisy at night but overall okay.",
|
| 95 |
+
"Good location but the apartment was smaller than expected.",
|
| 96 |
+
"It was fine for the price. Nothing special but clean enough.",
|
| 97 |
+
"Average stay. Check-in was smooth but wifi was slow.",
|
| 98 |
+
"The place served its purpose. Wouldn't say it was amazing though.",
|
| 99 |
+
"Okay for a short stay. The bathroom could use some updating.",
|
| 100 |
+
]
|
| 101 |
+
review_templates_negative = [
|
| 102 |
+
"Disappointed. The photos were misleading and it was dirty.",
|
| 103 |
+
"Terrible experience. Host was unresponsive and place was filthy.",
|
| 104 |
+
"Would not recommend. Noisy neighbors and broken appliances.",
|
| 105 |
+
"Not worth the price at all. Bed was uncomfortable.",
|
| 106 |
+
"Very poorly maintained. Found bugs in the kitchen area.",
|
| 107 |
+
"Host cancelled last minute. Terrible communication throughout.",
|
| 108 |
+
]
|
| 109 |
+
|
| 110 |
+
n_reviews = 5000
|
| 111 |
+
review_listing_ids = np.random.choice(listings['listing_id'], n_reviews)
|
| 112 |
+
|
| 113 |
+
# Bias reviews based on listing rating
|
| 114 |
+
reviews_list = []
|
| 115 |
+
for lid in review_listing_ids:
|
| 116 |
+
rating = listings.loc[listings['listing_id'] == lid, 'review_scores_rating'].values[0]
|
| 117 |
+
if rating >= 4.5:
|
| 118 |
+
probs = [0.75, 0.2, 0.05]
|
| 119 |
+
elif rating >= 4.0:
|
| 120 |
+
probs = [0.5, 0.35, 0.15]
|
| 121 |
+
else:
|
| 122 |
+
probs = [0.25, 0.35, 0.4]
|
| 123 |
+
|
| 124 |
+
category = np.random.choice(['positive', 'neutral', 'negative'], p=probs)
|
| 125 |
+
if category == 'positive':
|
| 126 |
+
text = np.random.choice(review_templates_positive)
|
| 127 |
+
elif category == 'neutral':
|
| 128 |
+
text = np.random.choice(review_templates_neutral)
|
| 129 |
+
else:
|
| 130 |
+
text = np.random.choice(review_templates_negative)
|
| 131 |
+
|
| 132 |
+
reviews_list.append({
|
| 133 |
+
'listing_id': lid,
|
| 134 |
+
'date': pd.Timestamp('2023-01-01') + pd.Timedelta(days=int(np.random.uniform(0, 730))),
|
| 135 |
+
'comments': text
|
| 136 |
+
})
|
| 137 |
+
|
| 138 |
+
reviews = pd.DataFrame(reviews_list)
|
| 139 |
+
print(f"Generated {len(reviews)} reviews")
|
| 140 |
+
|
| 141 |
+
# βββ Generate synthetic bookings βββ
|
| 142 |
+
n_bookings = 3000
|
| 143 |
+
guest_types = ['Solo', 'Couple', 'Family', 'Business']
|
| 144 |
+
bookings = pd.DataFrame({
|
| 145 |
+
'booking_id': range(1, n_bookings + 1),
|
| 146 |
+
'listing_id': np.random.choice(listings['listing_id'], n_bookings),
|
| 147 |
+
'booking_date': pd.date_range('2023-01-01', periods=n_bookings, freq='4h')[:n_bookings],
|
| 148 |
+
'length_of_stay': np.random.choice([1,2,3,4,5,7,14], n_bookings, p=[0.15,0.2,0.25,0.15,0.1,0.1,0.05]),
|
| 149 |
+
'guest_type': np.random.choice(guest_types, n_bookings, p=[0.2,0.35,0.25,0.2]),
|
| 150 |
+
'cancellation': np.random.choice([0,1], n_bookings, p=[0.85,0.15]),
|
| 151 |
+
})
|
| 152 |
+
bookings['satisfaction_score'] = np.clip(np.random.normal(7.5, 1.5, n_bookings), 1, 10).round(1)
|
| 153 |
+
print(f"Generated {len(bookings)} synthetic bookings")
|
| 154 |
+
|
| 155 |
+
# βββ Save raw datasets βββ
|
| 156 |
+
listings.to_csv(f"{OUTPUT_DIR}/listings_clean.csv", index=False)
|
| 157 |
+
reviews.to_csv(f"{OUTPUT_DIR}/reviews_clean.csv", index=False)
|
| 158 |
+
bookings.to_csv(f"{OUTPUT_DIR}/bookings_synthetic.csv", index=False)
|
| 159 |
+
print("Datasets saved.\n")
|
| 160 |
+
|
| 161 |
+
# ==============================================================
|
| 162 |
+
print("=" * 60)
|
| 163 |
+
print("PHASE 2: QUALITATIVE ANALYSIS β VADER Sentiment")
|
| 164 |
+
print("=" * 60)
|
| 165 |
+
|
| 166 |
+
analyzer = SentimentIntensityAnalyzer()
|
| 167 |
+
reviews['sentiment_compound'] = reviews['comments'].apply(
|
| 168 |
+
lambda x: analyzer.polarity_scores(str(x))['compound']
|
| 169 |
+
)
|
| 170 |
+
reviews['sentiment_label'] = reviews['sentiment_compound'].apply(
|
| 171 |
+
lambda x: 'Positive' if x >= 0.05 else ('Negative' if x <= -0.05 else 'Neutral')
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
print(f"\nSentiment Distribution:")
|
| 175 |
+
print(reviews['sentiment_label'].value_counts().to_string())
|
| 176 |
+
|
| 177 |
+
# Aggregate sentiment per listing
|
| 178 |
+
listing_sentiment = reviews.groupby('listing_id').agg(
|
| 179 |
+
avg_sentiment=('sentiment_compound', 'mean'),
|
| 180 |
+
review_count=('sentiment_compound', 'count'),
|
| 181 |
+
pct_positive=('sentiment_label', lambda x: (x == 'Positive').mean()),
|
| 182 |
+
pct_negative=('sentiment_label', lambda x: (x == 'Negative').mean()),
|
| 183 |
+
).reset_index()
|
| 184 |
+
|
| 185 |
+
# Merge sentiment into listings
|
| 186 |
+
listings = listings.merge(listing_sentiment, on='listing_id', how='left')
|
| 187 |
+
listings['avg_sentiment'] = listings['avg_sentiment'].fillna(0)
|
| 188 |
+
|
| 189 |
+
# βββ CHART 1: Sentiment by Neighbourhood βββ
|
| 190 |
+
fig, ax = plt.subplots(figsize=(12, 6))
|
| 191 |
+
neighbourhood_sentiment = listings.groupby('neighbourhood')['avg_sentiment'].mean().sort_values(ascending=True)
|
| 192 |
+
colors = ['#e74c3c' if v < 0.2 else '#f39c12' if v < 0.4 else '#27ae60' for v in neighbourhood_sentiment]
|
| 193 |
+
neighbourhood_sentiment.plot(kind='barh', ax=ax, color=colors, edgecolor='white', linewidth=0.5)
|
| 194 |
+
ax.set_xlabel('Average Sentiment Score', fontsize=12)
|
| 195 |
+
ax.set_ylabel('')
|
| 196 |
+
ax.set_title('Average Guest Sentiment by Neighbourhood', fontsize=14, fontweight='bold')
|
| 197 |
+
ax.axvline(x=neighbourhood_sentiment.mean(), color='#2c3e50', linestyle='--', alpha=0.7, label='City Average')
|
| 198 |
+
ax.legend()
|
| 199 |
+
plt.tight_layout()
|
| 200 |
+
plt.savefig(f"{OUTPUT_DIR}/chart1_sentiment_by_neighbourhood.png", dpi=150, bbox_inches='tight')
|
| 201 |
+
plt.close()
|
| 202 |
+
print("Chart 1 saved: Sentiment by Neighbourhood")
|
| 203 |
+
|
| 204 |
+
# βββ CHART 2: Price vs Sentiment Scatter βββ
|
| 205 |
+
fig, ax = plt.subplots(figsize=(10, 6))
|
| 206 |
+
scatter = ax.scatter(listings['price'], listings['avg_sentiment'],
|
| 207 |
+
c=listings['review_scores_rating'], cmap='RdYlGn',
|
| 208 |
+
alpha=0.6, s=40, edgecolors='gray', linewidth=0.3)
|
| 209 |
+
plt.colorbar(scatter, label='Review Score Rating')
|
| 210 |
+
ax.set_xlabel('Price (β¬/night)', fontsize=12)
|
| 211 |
+
ax.set_ylabel('Average Sentiment Score', fontsize=12)
|
| 212 |
+
ax.set_title('Price vs. Guest Sentiment (colored by rating)', fontsize=14, fontweight='bold')
|
| 213 |
+
plt.tight_layout()
|
| 214 |
+
plt.savefig(f"{OUTPUT_DIR}/chart2_price_vs_sentiment.png", dpi=150, bbox_inches='tight')
|
| 215 |
+
plt.close()
|
| 216 |
+
print("Chart 2 saved: Price vs Sentiment")
|
| 217 |
+
|
| 218 |
+
# βββ CHART 3: Sentiment Distribution βββ
|
| 219 |
+
fig, ax = plt.subplots(figsize=(10, 5))
|
| 220 |
+
sentiment_counts = reviews['sentiment_label'].value_counts()
|
| 221 |
+
colors_pie = ['#27ae60', '#f39c12', '#e74c3c']
|
| 222 |
+
sentiment_counts.plot(kind='bar', ax=ax, color=colors_pie, edgecolor='white', linewidth=1.5)
|
| 223 |
+
ax.set_ylabel('Number of Reviews', fontsize=12)
|
| 224 |
+
ax.set_title('Overall Review Sentiment Distribution', fontsize=14, fontweight='bold')
|
| 225 |
+
ax.set_xticklabels(ax.get_xticklabels(), rotation=0)
|
| 226 |
+
for i, v in enumerate(sentiment_counts):
|
| 227 |
+
ax.text(i, v + 30, f'{v} ({v/len(reviews)*100:.1f}%)', ha='center', fontweight='bold')
|
| 228 |
+
plt.tight_layout()
|
| 229 |
+
plt.savefig(f"{OUTPUT_DIR}/chart3_sentiment_distribution.png", dpi=150, bbox_inches='tight')
|
| 230 |
+
plt.close()
|
| 231 |
+
print("Chart 3 saved: Sentiment Distribution")
|
| 232 |
+
|
| 233 |
+
# βββ CHART 4: Superhost vs Non-Superhost Sentiment βββ
|
| 234 |
+
fig, ax = plt.subplots(figsize=(8, 5))
|
| 235 |
+
superhost_data = listings.groupby('host_is_superhost')['avg_sentiment'].mean()
|
| 236 |
+
superhost_data.index = ['Regular Host', 'Superhost']
|
| 237 |
+
superhost_data.plot(kind='bar', ax=ax, color=['#3498db', '#e67e22'], edgecolor='white', linewidth=1.5)
|
| 238 |
+
ax.set_ylabel('Average Sentiment Score', fontsize=12)
|
| 239 |
+
ax.set_title('Superhost vs Regular Host: Guest Sentiment', fontsize=14, fontweight='bold')
|
| 240 |
+
ax.set_xticklabels(ax.get_xticklabels(), rotation=0)
|
| 241 |
+
for i, v in enumerate(superhost_data):
|
| 242 |
+
ax.text(i, v + 0.005, f'{v:.3f}', ha='center', fontweight='bold')
|
| 243 |
+
plt.tight_layout()
|
| 244 |
+
plt.savefig(f"{OUTPUT_DIR}/chart4_superhost_sentiment.png", dpi=150, bbox_inches='tight')
|
| 245 |
+
plt.close()
|
| 246 |
+
print("Chart 4 saved: Superhost vs Regular Sentiment\n")
|
| 247 |
+
|
| 248 |
+
# ==============================================================
|
| 249 |
+
print("=" * 60)
|
| 250 |
+
print("PHASE 3A: QUANTITATIVE ANALYSIS β Random Forest Classification")
|
| 251 |
+
print("=" * 60)
|
| 252 |
+
|
| 253 |
+
# Create target variable: HighPerformer
|
| 254 |
+
median_rating = listings['review_scores_rating'].median()
|
| 255 |
+
median_reviews = listings['reviews_per_month'].median()
|
| 256 |
+
listings['HighPerformer'] = ((listings['review_scores_rating'] >= median_rating) &
|
| 257 |
+
(listings['reviews_per_month'] >= median_reviews)).astype(int)
|
| 258 |
+
|
| 259 |
+
print(f"\nTarget variable distribution:")
|
| 260 |
+
print(f" High Performers: {listings['HighPerformer'].sum()} ({listings['HighPerformer'].mean()*100:.1f}%)")
|
| 261 |
+
print(f" Low Performers: {(1-listings['HighPerformer']).sum()} ({(1-listings['HighPerformer']).mean()*100:.1f}%)")
|
| 262 |
+
|
| 263 |
+
# Prepare features
|
| 264 |
+
le = LabelEncoder()
|
| 265 |
+
listings['room_type_encoded'] = le.fit_transform(listings['room_type'])
|
| 266 |
+
listings['neighbourhood_encoded'] = le.fit_transform(listings['neighbourhood'])
|
| 267 |
+
|
| 268 |
+
feature_cols = ['price', 'accommodates', 'bedrooms', 'minimum_nights',
|
| 269 |
+
'number_of_reviews', 'host_is_superhost', 'instant_bookable',
|
| 270 |
+
'avg_sentiment', 'room_type_encoded', 'neighbourhood_encoded']
|
| 271 |
+
|
| 272 |
+
X = listings[feature_cols].fillna(0)
|
| 273 |
+
y = listings['HighPerformer']
|
| 274 |
+
|
| 275 |
+
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
|
| 276 |
+
|
| 277 |
+
rf = RandomForestClassifier(n_estimators=100, random_state=42, max_depth=10)
|
| 278 |
+
rf.fit(X_train, y_train)
|
| 279 |
+
y_pred = rf.predict(X_test)
|
| 280 |
+
|
| 281 |
+
print("\nClassification Report:")
|
| 282 |
+
report = classification_report(y_test, y_pred, target_names=['Low Performer', 'High Performer'])
|
| 283 |
+
print(report)
|
| 284 |
+
|
| 285 |
+
# Save classification report to file
|
| 286 |
+
with open(f"{OUTPUT_DIR}/classification_report.txt", 'w') as f:
|
| 287 |
+
f.write("RANDOM FOREST CLASSIFICATION REPORT\n")
|
| 288 |
+
f.write("=" * 50 + "\n")
|
| 289 |
+
f.write(f"Training set: {len(X_train)} listings\n")
|
| 290 |
+
f.write(f"Test set: {len(X_test)} listings\n\n")
|
| 291 |
+
f.write(report)
|
| 292 |
+
|
| 293 |
+
# βββ CHART 5: Feature Importance βββ
|
| 294 |
+
importances = pd.Series(rf.feature_importances_, index=feature_cols).sort_values(ascending=True)
|
| 295 |
+
fig, ax = plt.subplots(figsize=(10, 6))
|
| 296 |
+
importances.plot(kind='barh', ax=ax, color='#3498db', edgecolor='white', linewidth=0.5)
|
| 297 |
+
ax.set_xlabel('Feature Importance', fontsize=12)
|
| 298 |
+
ax.set_title('Random Forest: Feature Importance for Listing Performance', fontsize=14, fontweight='bold')
|
| 299 |
+
plt.tight_layout()
|
| 300 |
+
plt.savefig(f"{OUTPUT_DIR}/chart5_feature_importance.png", dpi=150, bbox_inches='tight')
|
| 301 |
+
plt.close()
|
| 302 |
+
print("Chart 5 saved: Feature Importance")
|
| 303 |
+
|
| 304 |
+
# βββ CHART 6: Confusion Matrix βββ
|
| 305 |
+
fig, ax = plt.subplots(figsize=(7, 6))
|
| 306 |
+
cm = confusion_matrix(y_test, y_pred)
|
| 307 |
+
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=ax,
|
| 308 |
+
xticklabels=['Low Performer', 'High Performer'],
|
| 309 |
+
yticklabels=['Low Performer', 'High Performer'])
|
| 310 |
+
ax.set_xlabel('Predicted', fontsize=12)
|
| 311 |
+
ax.set_ylabel('Actual', fontsize=12)
|
| 312 |
+
ax.set_title('Confusion Matrix', fontsize=14, fontweight='bold')
|
| 313 |
+
plt.tight_layout()
|
| 314 |
+
plt.savefig(f"{OUTPUT_DIR}/chart6_confusion_matrix.png", dpi=150, bbox_inches='tight')
|
| 315 |
+
plt.close()
|
| 316 |
+
print("Chart 6 saved: Confusion Matrix\n")
|
| 317 |
+
|
| 318 |
+
# ==============================================================
|
| 319 |
+
print("=" * 60)
|
| 320 |
+
print("PHASE 3B: QUANTITATIVE ANALYSIS β ARIMA Forecasting")
|
| 321 |
+
print("=" * 60)
|
| 322 |
+
|
| 323 |
+
# Create monthly average price by neighbourhood
|
| 324 |
+
listings['month_created'] = pd.to_datetime('2023-01-01') + pd.to_timedelta(
|
| 325 |
+
np.random.randint(0, 730, len(listings)), unit='D'
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
# Generate monthly time series per neighbourhood (simulate 24 months)
|
| 329 |
+
months = pd.date_range('2023-01-01', periods=24, freq='MS')
|
| 330 |
+
top_neighbourhoods = listings['neighbourhood'].value_counts().head(5).index.tolist()
|
| 331 |
+
|
| 332 |
+
fig, axes = plt.subplots(len(top_neighbourhoods), 1, figsize=(12, 4*len(top_neighbourhoods)))
|
| 333 |
+
|
| 334 |
+
forecast_results = {}
|
| 335 |
+
|
| 336 |
+
for idx, neighbourhood in enumerate(top_neighbourhoods):
|
| 337 |
+
# Generate realistic time series with trend and seasonality
|
| 338 |
+
base = listings[listings['neighbourhood'] == neighbourhood]['price'].mean()
|
| 339 |
+
trend = np.linspace(0, base * 0.15, 24) # slight upward trend
|
| 340 |
+
seasonality = base * 0.1 * np.sin(np.linspace(0, 4*np.pi, 24)) # seasonal pattern
|
| 341 |
+
noise = np.random.normal(0, base * 0.03, 24)
|
| 342 |
+
ts = base + trend + seasonality + noise
|
| 343 |
+
|
| 344 |
+
series = pd.Series(ts, index=months)
|
| 345 |
+
|
| 346 |
+
# Fit ARIMA(1,1,1)
|
| 347 |
+
try:
|
| 348 |
+
model = ARIMA(series, order=(1,1,1))
|
| 349 |
+
fitted = model.fit()
|
| 350 |
+
forecast = fitted.forecast(steps=6)
|
| 351 |
+
forecast_index = pd.date_range(months[-1] + pd.DateOffset(months=1), periods=6, freq='MS')
|
| 352 |
+
|
| 353 |
+
forecast_results[neighbourhood] = {
|
| 354 |
+
'historical': series,
|
| 355 |
+
'forecast': pd.Series(forecast.values, index=forecast_index),
|
| 356 |
+
'base_price': base
|
| 357 |
+
}
|
| 358 |
+
|
| 359 |
+
# Plot
|
| 360 |
+
ax = axes[idx]
|
| 361 |
+
ax.plot(series.index, series.values, 'b-o', markersize=3, label='Historical', linewidth=1.5)
|
| 362 |
+
ax.plot(forecast_index, forecast.values, 'r--o', markersize=3, label='Forecast (6 months)', linewidth=1.5)
|
| 363 |
+
ax.fill_between(forecast_index, forecast.values * 0.9, forecast.values * 1.1,
|
| 364 |
+
alpha=0.2, color='red', label='Confidence band')
|
| 365 |
+
ax.set_title(f'{neighbourhood} β Average Price Forecast', fontsize=12, fontweight='bold')
|
| 366 |
+
ax.set_ylabel('Price (β¬)')
|
| 367 |
+
ax.legend(loc='upper left', fontsize=8)
|
| 368 |
+
ax.grid(True, alpha=0.3)
|
| 369 |
+
|
| 370 |
+
print(f" {neighbourhood}: Current avg β¬{base:.0f} β Forecasted β¬{forecast.values[-1]:.0f} (6mo)")
|
| 371 |
+
except Exception as e:
|
| 372 |
+
print(f" {neighbourhood}: ARIMA failed - {e}")
|
| 373 |
+
|
| 374 |
+
plt.suptitle('ARIMA(1,1,1) Price Forecasting by Neighbourhood', fontsize=14, fontweight='bold', y=1.01)
|
| 375 |
+
plt.tight_layout()
|
| 376 |
+
plt.savefig(f"{OUTPUT_DIR}/chart7_arima_forecasts.png", dpi=150, bbox_inches='tight')
|
| 377 |
+
plt.close()
|
| 378 |
+
print("Chart 7 saved: ARIMA Forecasts\n")
|
| 379 |
+
|
| 380 |
+
# βββ CHART 8: Price distribution by room type βββ
|
| 381 |
+
fig, ax = plt.subplots(figsize=(10, 6))
|
| 382 |
+
room_type_order = ['Entire home/apt', 'Private room', 'Shared room']
|
| 383 |
+
listings.boxplot(column='price', by='room_type', ax=ax,
|
| 384 |
+
positions=[1,2,3] if len(listings['room_type'].unique()) == 3 else None)
|
| 385 |
+
ax.set_title('Price Distribution by Room Type', fontsize=14, fontweight='bold')
|
| 386 |
+
ax.set_xlabel('Room Type', fontsize=12)
|
| 387 |
+
ax.set_ylabel('Price (β¬/night)', fontsize=12)
|
| 388 |
+
plt.suptitle('')
|
| 389 |
+
plt.tight_layout()
|
| 390 |
+
plt.savefig(f"{OUTPUT_DIR}/chart8_price_by_room_type.png", dpi=150, bbox_inches='tight')
|
| 391 |
+
plt.close()
|
| 392 |
+
print("Chart 8 saved: Price by Room Type")
|
| 393 |
+
|
| 394 |
+
# βββ CHART 9: Booking patterns (synthetic data analysis) βββ
|
| 395 |
+
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
|
| 396 |
+
|
| 397 |
+
# Guest type distribution
|
| 398 |
+
guest_counts = bookings['guest_type'].value_counts()
|
| 399 |
+
axes[0].pie(guest_counts, labels=guest_counts.index, autopct='%1.1f%%',
|
| 400 |
+
colors=['#3498db','#e67e22','#27ae60','#9b59b6'], startangle=90)
|
| 401 |
+
axes[0].set_title('Booking Distribution by Guest Type', fontsize=12, fontweight='bold')
|
| 402 |
+
|
| 403 |
+
# Satisfaction by guest type
|
| 404 |
+
bookings.groupby('guest_type')['satisfaction_score'].mean().sort_values().plot(
|
| 405 |
+
kind='barh', ax=axes[1], color='#3498db', edgecolor='white')
|
| 406 |
+
axes[1].set_xlabel('Average Satisfaction Score (1-10)')
|
| 407 |
+
axes[1].set_title('Satisfaction Score by Guest Type', fontsize=12, fontweight='bold')
|
| 408 |
+
|
| 409 |
+
plt.tight_layout()
|
| 410 |
+
plt.savefig(f"{OUTPUT_DIR}/chart9_booking_patterns.png", dpi=150, bbox_inches='tight')
|
| 411 |
+
plt.close()
|
| 412 |
+
print("Chart 9 saved: Booking Patterns\n")
|
| 413 |
+
|
| 414 |
+
# ==============================================================
|
| 415 |
+
print("=" * 60)
|
| 416 |
+
print("KEY FINDINGS SUMMARY")
|
| 417 |
+
print("=" * 60)
|
| 418 |
+
|
| 419 |
+
# Top features
|
| 420 |
+
top_features = importances.tail(3).index.tolist()
|
| 421 |
+
print(f"\n1. Top 3 predictive features for listing performance:")
|
| 422 |
+
for i, f in enumerate(reversed(top_features)):
|
| 423 |
+
print(f" {i+1}. {f} (importance: {importances[f]:.3f})")
|
| 424 |
+
|
| 425 |
+
# Best/worst neighbourhoods
|
| 426 |
+
best_hood = neighbourhood_sentiment.idxmax()
|
| 427 |
+
worst_hood = neighbourhood_sentiment.idxmin()
|
| 428 |
+
print(f"\n2. Neighbourhood insights:")
|
| 429 |
+
print(f" Highest sentiment: {best_hood} ({neighbourhood_sentiment.max():.3f})")
|
| 430 |
+
print(f" Lowest sentiment: {worst_hood} ({neighbourhood_sentiment.min():.3f})")
|
| 431 |
+
|
| 432 |
+
# Superhost effect
|
| 433 |
+
sh_sent = listings[listings['host_is_superhost']==1]['avg_sentiment'].mean()
|
| 434 |
+
nsh_sent = listings[listings['host_is_superhost']==0]['avg_sentiment'].mean()
|
| 435 |
+
print(f"\n3. Superhost effect:")
|
| 436 |
+
print(f" Superhost avg sentiment: {sh_sent:.3f}")
|
| 437 |
+
print(f" Regular host avg sentiment: {nsh_sent:.3f}")
|
| 438 |
+
print(f" Difference: +{sh_sent - nsh_sent:.3f} for superhosts")
|
| 439 |
+
|
| 440 |
+
# Sentiment breakdown
|
| 441 |
+
pos_pct = (reviews['sentiment_label'] == 'Positive').mean() * 100
|
| 442 |
+
neg_pct = (reviews['sentiment_label'] == 'Negative').mean() * 100
|
| 443 |
+
print(f"\n4. Review sentiment breakdown:")
|
| 444 |
+
print(f" Positive: {pos_pct:.1f}%")
|
| 445 |
+
print(f" Negative: {neg_pct:.1f}%")
|
| 446 |
+
|
| 447 |
+
# Forecast
|
| 448 |
+
print(f"\n5. Price forecast highlights (next 6 months):")
|
| 449 |
+
for hood, data in forecast_results.items():
|
| 450 |
+
last_hist = data['historical'].iloc[-1]
|
| 451 |
+
last_fore = data['forecast'].iloc[-1]
|
| 452 |
+
change = ((last_fore - last_hist) / last_hist) * 100
|
| 453 |
+
print(f" {hood}: β¬{last_hist:.0f} β β¬{last_fore:.0f} ({change:+.1f}%)")
|
| 454 |
+
|
| 455 |
+
# Save master dataset
|
| 456 |
+
listings.to_csv(f"{OUTPUT_DIR}/master_listings_analyzed.csv", index=False)
|
| 457 |
+
reviews.to_csv(f"{OUTPUT_DIR}/reviews_with_sentiment.csv", index=False)
|
| 458 |
+
print(f"\nAll outputs saved to {OUTPUT_DIR}/")
|
| 459 |
+
print("=" * 60)
|
| 460 |
+
print("ANALYSIS COMPLETE")
|
| 461 |
+
print("=" * 60)
|