import pandas as pd
import numpy as np

# Load the dataset
df = pd.read_csv('housing_data.csv')

# Inspect raw data types and missing values
df.info()

# View summary statistics
display(df.describe())

from sklearn.impute import SimpleImputer, KNNImputer

# 1. Simple Imputation (Mean/Median/Most Frequent)
# Good for MCAR (Missing Completely At Random)
mean_imputer = SimpleImputer(strategy='mean')
df['age_imputed'] = mean_imputer.fit_transform(df[['age']])

# 2. KNN Imputation (Distance-based)
# Good for MAR (Missing At Random) when variables are correlated
knn_imputer = KNNImputer(n_neighbors=5, weights='distance')
df_imputed = knn_imputer.fit_transform(df)

# Note: Tree-based models like XGBoost can handle NaNs natively!

import numpy as np
from scipy import stats

# 1. Z-Score Method (Dropping Outliers)
z_scores = np.abs(stats.zscore(df['income']))
# Keep only rows where z-score is less than 3
df_clean = df[z_scores < 3]

# 2. IQR Method (Winsorization / Capping)
# Capping at 5th and 95th percentiles to retain data points
lower_limit = df['income'].quantile(0.05)
upper_limit = df['income'].quantile(0.95)

df['income_capped'] = np.clip(df['income'], lower_limit, upper_limit)

from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler

# 1. Min-Max Scaler (Best for Neural Networks/Images)
minmax = MinMaxScaler()
df[['age_minmax', 'income_minmax']] = minmax.fit_transform(df[['age', 'income']])

# 2. Standard Scaler (Best for PCA, SVM, Logistic Regression)
standard = StandardScaler()
df_scaled = standard.fit_transform(df)

# 3. Robust Scaler (Best when dataset has many outliers)
robust = RobustScaler()
df_robust = robust.fit_transform(df)

import pandas as pd
from sklearn.preprocessing import OneHotEncoder
from category_encoders import TargetEncoder

# 1. One-Hot Encoding (Best for nominal variables with few categories)
ohe = OneHotEncoder(sparse_output=False, drop='first') # drop='first' avoids multicollinearity
color_encoded = ohe.fit_transform(df[['color']])

# Pandas alternative (easy but not ideal for pipelines):
# pd.get_dummies(df, columns=['color'], drop_first=True)

# 2. Target Encoding (Best for high-cardinality nominal variables like zipcodes)
# Requires 'category_encoders' library
te = TargetEncoder(smoothing=10) # Higher smoothing pulls estimates closer to global mean
df['zipcode_encoded'] = te.fit_transform(df['zipcode'], df['target'])

from sklearn.feature_selection import SelectKBest, f_classif, RFE
from sklearn.linear_model import LogisticRegression

X = df.drop('target', axis=1)
y = df['target']

# 1. Filter Method: SelectKBest (ANOVA F-value)
# Keeps the 5 features with the highest ANOVA F-scores
selector = SelectKBest(score_func=f_classif, k=5)
X_top_5 = selector.fit_transform(X, y)
selected_columns = X.columns[selector.get_support()]

# 2. Wrapper Method: Recursive Feature Elimination (RFE)
# Uses a model's intrinsic feature importance assigning to prune
estimator = LogisticRegression()
rfe = RFE(estimator, n_features_to_select=5, step=1)
X_rfe = rfe.fit_transform(X, y)
rfe_columns = X.columns[rfe.support_]

from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler
from imblearn.pipeline import Pipeline

# 1. SMOTE (Over-sampling the minority class)
smote = SMOTE(sampling_strategy='auto', k_neighbors=5, random_state=42)
X_smote, y_smote = smote.fit_resample(X, y)

# 2. Random Under-Sampling (Reducing the majority class)
rus = RandomUnderSampler(sampling_strategy='auto', random_state=42)
X_rus, y_rus = rus.fit_resample(X, y)

# 3. Best Practice Pipeline: Under-sample majority THEN SMOTE minority
# Prevents creating too many synthetic points if the imbalance is extreme
resample_pipe = Pipeline([
    ('rus', RandomUnderSampler(sampling_strategy=0.1)), # Reduce majority until minority is 10%
    ('smote', SMOTE(sampling_strategy=0.5))             # SMOTE minority until it's 50%
])
X_resampled, y_resampled = resample_pipe.fit_resample(X, y)

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# 1. Deep Descriptive Stats (includes skewness)
display(df.describe().T)
print("Skewness:\n", df.skew())
print("\nMissing Values:\n", df.isnull().sum())

# 2. Visual Distributions (Pairplot)
# Plots histograms on the diagonal and scatter plots for every relationship
sns.pairplot(df, hue='target_class', diag_kind='kde', corner=True)
plt.show()

# 3. Correlation Heatmap
plt.figure(figsize=(10, 8))
corr_matrix = df.corr(method='spearman') # Spearman is robust to non-linear relationships
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', vmin=-1, vmax=1)
plt.title("Spearman Correlation Heatmap")
plt.show()

import numpy as np
from sklearn.preprocessing import PowerTransformer, KBinsDiscretizer

# 1. Power Transformation (Yeo-Johnson)
# Attempts to map skewed feature to a Gaussian distribution
pt = PowerTransformer(method='yeo-johnson', standardize=True)
df['income_gaussian'] = pt.fit_transform(df[['income']])

# 2. Log Transformation (np.log1p handles zeros safely by doing log(1+x))
df['revenue_log'] = np.log1p(df['revenue'])

# 3. Discretization / Binning
# Converts continuous age into 5 categorical bins (quantiles ensures equal frequency per bin)
binner = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='quantile')
df['age_group'] = binner.fit_transform(df[['age']])

from sklearn.preprocessing import PolynomialFeatures
import pandas as pd

# Assume df has two features: 'length' and 'width'
X = df[['length', 'width']]

# Create polynomial and interaction features up to degree 2
# include_bias=False prevents adding a column of 1s (intercept)
poly = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly.fit_transform(X)

# Get the names of the new features (e.g., 'length^2', 'length width')
feature_names = poly.get_feature_names_out(['length', 'width'])
df_poly = pd.DataFrame(X_poly, columns=feature_names)

print(df_poly.head())

from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import numpy as np

# 1. ALWAYS scale data before PCA
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# 2. Fit PCA without specifying components to see all variance
pca_full = PCA()
pca_full.fit(X_scaled)

# 3. Plot Cumulative Explained Variance
cumulative_variance = np.cumsum(pca_full.explained_variance_ratio_)
plt.plot(cumulative_variance, marker='o')
plt.axhline(y=0.95, color='r', linestyle='--') # 95% threshold
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Explained Variance')
plt.show()

# 4. Apply PCA retaining 95% variance
# Float between 0 and 1 selects components covering that % of variance
pca = PCA(n_components=0.95)
X_pca = pca.fit_transform(X_scaled)
print(f"Reduced from {X.shape[1]} to {X_pca.shape[1]} features.")

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import pandas as pd

# Sample text column
corpus = [
    "Machine learning is amazing",
    "Deep learning is the future of learning",
    "Data science and artificial intelligence"
]

# 1. Bag of Words (CountVectorizer)
# Creates a column for every unique word in the corpus
vectorizer = CountVectorizer(stop_words='english')
X_bow = vectorizer.fit_transform(corpus)

# 2. TF-IDF (TfidfVectorizer)
# Converts words to continuous weights between 0 and 1
tfidf = TfidfVectorizer(stop_words='english', max_features=100)
X_tfidf = tfidf.fit_transform(corpus)

# Quick way to view features as a DataFrame
tfidf_df = pd.DataFrame(X_tfidf.toarray(), columns=tfidf.get_feature_names_out())
print(tfidf_df.head())

import pandas as pd

# Assuming 'df' is sorted chronologically and indexed by Date
# 1. Lag Features (Looking back in time)
# What was the value 1 day ago? 7 days ago?
df['sales_lag_1'] = df['sales'].shift(1)
df['sales_lag_7'] = df['sales'].shift(7)

# 2. Rolling Window Features
# The average and standard deviation over the last 7 days
df['sales_rolling_mean_7d'] = df['sales'].rolling(window=7).mean()
df['sales_rolling_std_7d'] = df['sales'].rolling(window=7).std()

# 3. Expanding Window Features
# Year-to-date maximum sales
df['sales_expanding_max'] = df['sales'].expanding().max()

# Drop NaNs generated by shifting/rolling
df.dropna(inplace=True)

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# ❌ BAD PRACTICE (Creates Leakage)
scaler_bad = StandardScaler()
X_scaled_bad = scaler_bad.fit_transform(X) # Entire dataset sees the scaler
X_train_bad, X_test_bad = train_test_split(X_scaled_bad)

# ✅ GOOD PRACTICE (No Leakage)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

scaler = StandardScaler()
# Fit ONLY on the training data to learn parameters (mean, std)
X_train_scaled = scaler.fit_transform(X_train) 

# Transform test data using the parameters learned from the training data
X_test_scaled = scaler.transform(X_test)

import featuretools as ft

# Assume we have three Pandas DataFrames: clients, loans, and payments
# Step 1: Create an EntitySet (a representation of your database)
es = ft.EntitySet(id="banking")

# Step 2: Add dataframes to the EntitySet with primary keys
es = es.add_dataframe(dataframe_name="clients", dataframe=clients_df, index="client_id")
es = es.add_dataframe(dataframe_name="loans", dataframe=loans_df, index="loan_id")

# Step 3: Define relational joins (Foreign Keys)
es = es.add_relationship("clients", "client_id", "loans", "client_id")

# Step 4: Run Deep Feature Synthesis!
# Automatically generates agg features for clients based on their loans history
feature_matrix, feature_defs = ft.dfs(
    entityset=es,
    target_dataframe_name="clients",
    agg_primitives=["mean", "sum", "mode", "std"],
    trans_primitives=["month", "hour"],
    max_depth=2 # Stacks primitives up to 2 layers deep
)

print(f"Automatically generated {len(feature_defs)} features!")