""" ================================================================================ FEATURE ENGINEERING MODULE - Tabular Feature Extraction for ML Models ================================================================================ PURPOSE: Transforms raw transactions and graph topology into machine learning features. Combines network centrality scores, temporal patterns, and behavioral signals into a tabular matrix for XGBoost training and inference. FEATURE CATEGORIES: 1. TRANSACTION VOLUME (Behavioral Baseline) - tx_count_total: Total number of transactions sent - amount_sent_total: Sum of outgoing amounts - tx_count_7d: Transactions in last 7 days (recent activity) - amount_sent_7d: Outgoing amount last 7 days - amount_received_total: Sum of incoming amounts - amount_received_7d: Incoming amount last 7 days 2. TEMPORAL DYNAMICS (Activity Patterns) - first_tx: Date of first transaction (account age) - last_tx: Date of most recent transaction - days_since_last_tx: Inactive period indicator - velocity_ratio_7d: Recent / historical activity ratio 3. TRANSACTION CHARACTERISTICS (Sophistication) - avg_tx_amount: Mean transaction size - amount_std: Variance in transaction amounts - forward_ratio: Outgoing / incoming amount ratio - in_out_ratio: Incoming / outgoing ratio - channel_diversity: Count of unique payment types - bank_diversity: Count of unique counterparty banks - currency_diversity: Count of unique currencies used 4. NETWORK TOPOLOGY (Graph Importance) - pagerank_score: Centrality by transaction volume flow - betweenness_score: Intermediary role in network - community_fraud_rate: Fraud rate in local Louvain community 5. SUSPICIOUS PATTERN DETECTION (Anomaly Signals) - in_fraud_network: Binary: present in any is_laundering=1 edge - cycle_length: Longest round-tripping cycle (0 if none) - cycle_amount: Max amount in detected cycle - mule_network_score: Proximity to mule accounts FEATURE ENGINEERING PROCESS: 1. Pre-aggregate statistics (sent/received, total/7d windows) 2. Calculate diversity metrics (channels, banks, currencies) 3. Extract temporal features (first_tx, last_tx, velocity) 4. Compute network centrality (PageRank, betweenness, community detection) 5. Identify suspicious patterns (cycles, fraud networks) 6. Build DataFrame indexed by account, aligned for model input DESIGN DECISIONS: 1. Dual-Window Aggregation (Total + 7-day) - Captures both steady-state and recent behavior change - Enables detection of sudden activity shifts (pattern break) 2. Variance in Transaction Amounts - High variance suggests testing phase (micro-transactions) - Low variance suggests routine operations - Combined with std: standard deviation robust to outliers 3. Diversity Metrics - Channel/bank diversity indicates sophistication - Legitimate accounts reuse same channels - Fraudsters use many channels to obscure patterns 4. Community Target-Encoding - Louvain community fraud rate as node feature - Accounts in high-fraud communities are higher risk - More informative than raw community ID 5. Cycle Detection (vs Binary is_in_cycle) - Cycle length: longer cycles = more coordinated schemes - Cycle amount: higher amounts = more serious laundering - Better than binary flag for risk scoring 6. Forward Ratio & In/Out Ratio - Forward ratio >> 1: More sender than receiver (suspicious) - Forward ratio << 1: More receiver (may be deposit) - In/out ratio complements forward_ratio perspective DEPENDENCIES: - pandas: DataFrame operations and groupby aggregations - numpy: Numerical calculations - networkx (via graph_builder): Centrality scores - python-louvain: Community detection - src.config_loader: Configuration (window sizes, thresholds) HYPERPARAMETERS: - RECENT_WINDOW_DAYS = 7: Sliding window for recent activity USAGE EXAMPLE: features_df = engineer_features( df=transactions, G=transaction_graph, pagerank_scores=pr_scores, betweenness_scores=bc_scores, louvain_partition=communities, cycle_alerts=detected_cycles, ) # Returns: DataFrame with shape (num_accounts, num_features) # Index: account IDs # Columns: feature names (ready for ML model input) ================================================================================ """ import pandas as pd import numpy as np from datetime import datetime from src.config_loader import get_config RECENT_WINDOW_DAYS = 7 def engineer_features( df: pd.DataFrame, G, pagerank_scores: dict, betweenness_scores: dict, louvain_partition: dict, cycle_alerts: list, ) -> pd.DataFrame: cfg = get_config() df = df.copy() df['timestamp'] = pd.to_datetime(df['timestamp']) max_date = df['timestamp'].max() cutoff_7d = max_date - pd.Timedelta(days=RECENT_WINDOW_DAYS) now = pd.Timestamp(datetime.now()) # Collect all accounts that have been flagged in any is_laundering=1 edge fraud_sources = ( set(df[df['is_laundering'] == 1]['source'].unique()) | set(df[df['is_laundering'] == 1]['target'].unique()) ) all_accounts = set(df['source'].unique()) | set(df['target'].unique()) # Pre-aggregate transaction stats to avoid repeated groupby operations. # Separating total vs 7-day windows lets us capture recent behavior changes. sent_all = df.groupby('source').agg( tx_count_total=('amount', 'count'), amount_sent_total=('amount', 'sum'), ) sent_7d = df[df['timestamp'] >= cutoff_7d].groupby('source').agg( tx_count_7d=('amount', 'count'), amount_sent_7d=('amount', 'sum'), ) recv_all = df.groupby('target').agg( amount_received_total=('amount', 'sum'), ) recv_7d = df[df['timestamp'] >= cutoff_7d].groupby('target').agg( amount_received_7d=('amount', 'sum'), ) # Combine sent and received amounts to compute variance across all transactions. # High variance in transaction amounts is suspicious (often indicates testing phase). sent_amounts = df[['source', 'amount']].rename(columns={'source': 'account'}) recv_amounts = df[['target', 'amount']].rename(columns={'target': 'account'}) combined_amounts = pd.concat([sent_amounts, recv_amounts], ignore_index=True) amount_std = combined_amounts.groupby('account')['amount'].std().rename('amount_std') # Temporal features: account age and channel/bank diversity indicate sophistication first_tx = df.groupby('source')['timestamp'].min().rename('first_tx') last_tx = df.groupby('source')['timestamp'].max().rename('last_tx') channel_div = df.groupby('source')['payment_type'].nunique().rename('channel_diversity') bank_div = df.groupby('source')['target_bank'].nunique().rename('bank_diversity') \ if 'target_bank' in df.columns else pd.Series(dtype=float, name='bank_diversity') currency_div = df.groupby('source')['Payment Currency'].nunique().rename('currency_diversity') \ if 'Payment Currency' in df.columns else pd.Series(dtype=float, name='currency_diversity') # Extract cycle length and max involved amounts for accounts in detected round-tripping patterns. # We keep the longest cycle per account to capture the most sophisticated laundering scheme. cycle_length_map = {} cycle_amount_map = {} for alert in cycle_alerts: if alert.get('typology') == 'RoundTripping': members = alert.get('cycle_members', [alert['account']]) length = alert.get('tx_count', len(members)) amount = alert.get('amount_involved', 0) for m in members: if m not in cycle_length_map or length > cycle_length_map[m]: cycle_length_map[m] = length cycle_amount_map[m] = amount # Target-encode community membership: compute fraud rate per Louvain community. # Communities with high fraud rates are more suspicious and signal stronger risk. community_fraud = {} for node, cid in louvain_partition.items(): community_fraud.setdefault(cid, []).append(1 if node in fraud_sources else 0) community_fraud_rate = {cid: np.mean(vals) for cid, vals in community_fraud.items()} # Build feature DataFrame features = pd.DataFrame(index=list(all_accounts)) features.index.name = 'account' features = features.join(sent_all) features = features.join(sent_7d) features = features.join(recv_all) features = features.join(recv_7d) features['forward_ratio'] = features['amount_sent_total'] / (features['amount_received_total'] + 1) features['avg_tx_amount'] = features['amount_sent_total'] / (features['tx_count_total'] + 1) features = features.join(amount_std) features['in_out_ratio'] = features['amount_received_total'] / (features['amount_sent_total'] + 1) features['velocity_ratio_7d'] = features['tx_count_7d'] / (features['tx_count_total'] + 1) features['pagerank_score'] = features.index.map(lambda a: pagerank_scores.get(a, 0)) features['betweenness_score'] = features.index.map(lambda a: betweenness_scores.get(a, 0)) features['in_degree'] = features.index.map(lambda a: G.in_degree(a) if a in G else 0) features['out_degree'] = features.index.map(lambda a: G.out_degree(a) if a in G else 0) features['fan_in_ratio'] = features['in_degree'] / (features['in_degree'] + features['out_degree'] + 1) # FIX: target-encoded community (fraud rate per community) features['community_encoded'] = features.index.map( lambda a: community_fraud_rate.get(louvain_partition.get(a, -1), 0.0) ) # FIX: cycle_length + cycle_max_amount features['cycle_length'] = features.index.map(lambda a: cycle_length_map.get(a, 0)) features['cycle_max_amount'] = features.index.map(lambda a: cycle_amount_map.get(a, 0)) features = features.join(first_tx) features = features.join(last_tx) features['account_age_days'] = (features['last_tx'] - features['first_tx']).dt.days.fillna(0) features['days_since_last_tx'] = (now - features['last_tx']).dt.days.fillna(9999) features = features.drop(columns=['first_tx', 'last_tx'], errors='ignore') features = features.join(currency_div) features = features.join(channel_div) features = features.join(bank_div) features['fraud_flag'] = features.index.map(lambda a: 1 if a in fraud_sources else 0) features = features.replace([np.inf, -np.inf], 0).fillna(0) features = features.reset_index() return features