Upload train_v38_2_resid.py with huggingface_hub

train_v38_2_resid.py

@@ -1,13 +1,16 @@
 """
-V38.2 RESIDUALIZED MODEL - Simpson's Paradox Fix
 ====================================================================
-Key changes from bare model:
-1. Compute school_base_rate (time-respecting within each CV fold)
-2. Residualize all student-level features (subtract school-level mean)
-3. Add explicit student × school interaction features
-4. Keep original features too (let model choose)
-
-Architecture: CB + LGB + XGBoost 3-model blend, 3-seed x 10-fold GroupKFold (fast mode)
+V38.2-PRO-V4 MODEL - Restored Features + Supp Fix + v6 Feature Matrix
+====================================================================
+Changes from V38.2-PRO-V3:
+1. Use v6 feature matrix (v5 + restored v4 columns)
+2. Restored: hs_to_univ_hist_rate_smoothed (r=0.337!), ps_bert_pca_*, summer features
+3. Added honors_quality_ratio (max_score / (count+1))
+4. Supp score=1 (all dims=1) -> NaN (not real scores, just 'no supplement')
+5. has_ps=0 -> ps_bert_pca all NaN (fix false BERT signals)
+6. hs_to_univ_hist features: -1 -> NaN (already done in v6 CSV)
+7. All V3 fixes carried forward (SAT/TOEFL/GPA NaN, CatBoost native NaN)
+====================================================================
 """
 import pandas as pd
 import numpy as np
@@ -16,7 +19,6 @@ warnings.filterwarnings('ignore')
 from sklearn.model_selection import GroupKFold
 from sklearn.metrics import roc_auc_score, log_loss, brier_score_loss
 from sklearn.preprocessing import LabelEncoder
-from sklearn.calibration import calibration_curve
 from scipy.stats import rankdata
 
 try:
@@ -39,26 +41,44 @@ OUTPUT_DIR = os.path.join(BASE_DIR, 'output')
 os.makedirs(OUTPUT_DIR, exist_ok=True)
 
 TARGET = 'target'
-SEEDS = [42, 123, 456]
+SEEDS = [42, 123, 456, 789, 2024]
 N_FOLDS = 10
+FEATURE_SELECT_TOP_N = 150
 start_time = time.time()
 
-def safe_num(v, default=-1):
-    if isinstance(v, (int, float)): return float(v)
+def safe_num(v, default=np.nan):
+    """Convert to float, return NaN for missing (was -1 before)."""
+    if isinstance(v, (int, float)):
+        val = float(v)
+        return np.nan if val == -1 else val
     if isinstance(v, str):
-        try: return float(v)
-        except: return default
+        try:
+            val = float(v)
+            return np.nan if val == -1 else val
+        except:
+            return default
     return default
 
 # ============================================================
-# 1. LOAD DATA
+# 1. LOAD DATA (v6 feature matrix)
 # ============================================================
 print("=" * 70)
-print("  V38.2 RESIDUALIZED MODEL: SIMPSON'S PARADOX FIX")
+print("  V38.2-PRO-V4: RESTORED FEATURES + SUPP FIX + V6 MATRIX")
 print("=" * 70)
 
-df_raw = pd.read_csv(os.path.join(DATA_DIR, 'v38_2_integrated_features.csv'))
-print(f"Integrated features loaded: {df_raw.shape}")
+# Try v6 first, fall back to v5, then v4
+v6_path = os.path.join(DATA_DIR, 'v38_2_integrated_features_v6.csv')
+v5_path = os.path.join(DATA_DIR, 'v38_2_integrated_features_v5.csv')
+v4_path = os.path.join(DATA_DIR, 'v38_2_integrated_features.csv')
+if os.path.exists(v6_path):
+    df_raw = pd.read_csv(v6_path)
+    print(f"V6 features loaded: {df_raw.shape}")
+elif os.path.exists(v5_path):
+    df_raw = pd.read_csv(v5_path)
+    print(f"V5 features loaded (v6 not found): {df_raw.shape}")
+else:
+    df_raw = pd.read_csv(v4_path)
+    print(f"V4 features loaded: {df_raw.shape}")
 
 # Load LLM features
 llm_features_loaded = {}
@@ -76,16 +96,137 @@ for fname, varname in [
     else:
         llm_features_loaded[varname] = {}
 
+# Load raw data to get ED2 round info
+import re
+RAW_CSV = os.path.join(DATA_DIR, 'students_with_essays_merged_clean.csv')
+round_lookup = {}
+if os.path.exists(RAW_CSV):
+    print(f"\n  Loading raw CSV for ED2 round info...")
+    try:
+        raw_chunks = pd.read_csv(RAW_CSV, usecols=['student_id', 'school_results_summary'], 
+                                  dtype=str, chunksize=500)
+        for chunk in raw_chunks:
+            for _, row in chunk.iterrows():
+                sid = str(row.get('student_id', '')).replace('.0', '')
+                summary = str(row.get('school_results_summary', ''))
+                entries = re.split(r'(?=\d+\.)', summary)
+                for entry in entries:
+                    m = re.search(r'(Early Decision II|Early Decision|Early Action II|Early Action|Restrictive Early Action|Regular Decision)', entry)
+                    if m:
+                        round_type = m.group(1)
+                        school_m = re.search(r'\d+\.\s*(.+?)(?:\s*[-–]\s*|\s*\()', entry)
+                        if school_m:
+                            school_name = school_m.group(1).strip()
+                            key = f"{sid}_{school_name}"
+                            round_lookup[key] = round_type
+        print(f"  Round lookup built: {len(round_lookup)} entries")
+    except Exception as e:
+        print(f"  Warning: Could not load raw CSV: {e}")
+
 # ============================================================
-# 2. DATA PREPARATION
+# 2. DATA CLEANING & QUALITY FIXES
 # ============================================================
-df = df_raw[~df_raw['year'].isin([2018, 2019])].copy().reset_index(drop=True)
-print(f"\nAfter removing 2018-2019: {df.shape}")
-print(f"Years: {sorted(df['year'].unique())}")
-print(f"Admit rate: {df[TARGET].mean():.3f}")
+print(f"\n{'='*70}")
+print(f"  DATA QUALITY FIXES")
+print(f"{'='*70}")
+
+# 2a. Filter years
+df = df_raw[~df_raw['year'].isin([2018, 2019])].copy()
+df = df.reset_index(drop=True)
+print(f"After filtering 2018-2019: {df.shape}")
+
+# 2b. FIX #1: SAT=0 -> NaN + has_sat
+sat_zero = (df['sat'] == 0).sum()
+df['has_sat'] = (df['sat'] > 0).astype(int)
+df.loc[df['sat'] == 0, 'sat'] = np.nan
+print(f"\n  FIX #1: SAT=0 -> NaN: {sat_zero} rows ({sat_zero/len(df)*100:.1f}%)")
+print(f"    has_sat=1: {df['has_sat'].sum()}, has_sat=0: {(df['has_sat']==0).sum()}")
+
+# 2c. FIX #2: TOEFL=0 -> NaN + has_toefl
+toefl_zero = (df['toefl'] == 0).sum()
+df['has_toefl'] = (df['toefl'] > 0).astype(int)
+df.loc[df['toefl'] == 0, 'toefl'] = np.nan
+print(f"  FIX #2: TOEFL=0 -> NaN: {toefl_zero} rows ({toefl_zero/len(df)*100:.1f}%)")
+
+# 2d. FIX #3: GPA=0 -> NaN (v5 already has has_gpa)
+gpa_zero = (df['gpa'] == 0).sum()
+df.loc[df['gpa'] == 0, 'gpa'] = np.nan
+print(f"  FIX #3: GPA=0 -> NaN: {gpa_zero} rows ({gpa_zero/len(df)*100:.1f}%)")
+if 'has_gpa' not in df.columns:
+    df['has_gpa'] = df['gpa'].notna().astype(int)
+print(f"    has_gpa=1: {(df['has_gpa']==1).sum()}, has_gpa=0: {(df['has_gpa']==0).sum()}")
+
+# 2e. FIX #4: -1 -> NaN for sentinel columns
+sentinel_cols = ['taste_yearly_admits_log']
+# v5 removed hs_to_univ_hist_rate, hs_to_univ_hist_rate_smoothed, hs_overall_hist_rate
+# but check if they exist
+for col in ['hs_to_univ_hist_rate', 'hs_to_univ_hist_rate_smoothed', 'hs_overall_hist_rate']:
+    if col in df.columns:
+        sentinel_cols.append(col)
+
+for col in sentinel_cols:
+    if col in df.columns:
+        n_neg1 = (df[col] == -1).sum()
+        df.loc[df[col] == -1, col] = np.nan
+        print(f"  FIX #4: {col}: -1 -> NaN: {n_neg1} rows ({n_neg1/len(df)*100:.1f}%)")
+
+# 2f. FIX #5: has_ps=0 -> ps_bert all NaN
+act_bert_cols = [c for c in df.columns if c.startswith('act_bert_pca_')]
+# ps_bert_pca columns were removed in v5, but check
+ps_bert_cols = [c for c in df.columns if c.startswith('ps_bert_pca_')]
+if ps_bert_cols:
+    no_ps_mask = df['has_ps'] == 0
+    n_fix = no_ps_mask.sum()
+    for col in ps_bert_cols:
+        df.loc[no_ps_mask, col] = np.nan
+    print(f"  FIX #5: ps_bert -> NaN for has_ps=0: {n_fix} rows, {len(ps_bert_cols)} columns")
+else:
+    print(f"  FIX #5: No ps_bert_pca columns in v5 (already removed)")
+
+# 2g. FIX portfolio_size: log transform + cap (from V2)
+print(f"\n  Portfolio size transform:")
+print(f"    Before: mean={df['portfolio_size'].mean():.1f}, max={df['portfolio_size'].max():.0f}")
+df['portfolio_size_raw'] = df['portfolio_size'].copy()
+df['portfolio_size'] = np.log1p(df['portfolio_size'].clip(upper=20))
+print(f"    After log(clip(x,20)): mean={df['portfolio_size'].mean():.2f}, max={df['portfolio_size'].max():.2f}")
+df['portfolio_size_bin'] = pd.cut(df['portfolio_size_raw'], 
+                                   bins=[0, 5, 10, 15, 20, 100],
+                                   labels=[0, 1, 2, 3, 4]).astype(int)
+
+# 2h. ED2 split (from V2)
+def get_detailed_round(row):
+    sid = str(row.get('student_id', '')).replace('.0', '')
+    school = str(row.get('school', ''))
+    key = f"{sid}_{school}"
+    raw_round = round_lookup.get(key, '')
+    if 'Early Decision II' in raw_round:
+        return 'ED2'
+    elif 'Early Decision' in raw_round:
+        return 'ED1'
+    elif 'Restrictive Early Action' in raw_round:
+        return 'REA'
+    elif 'Early Action II' in raw_round or 'Early Action' in raw_round:
+        return 'EA'
+    elif 'Regular Decision' in raw_round:
+        return 'RD'
+    # Fall back to original round_cat
+    orig = str(row.get('round_cat', 'RD'))
+    if orig == 'ED':
+        return 'ED1'
+    return orig
+
+df['round_cat_v2'] = df.apply(get_detailed_round, axis=1)
+print(f"\n  Round distribution (v2):")
+print(df['round_cat_v2'].value_counts().to_string())
+
+df['is_ed1'] = (df['round_cat_v2'] == 'ED1').astype(int)
+df['is_ed2'] = (df['round_cat_v2'] == 'ED2').astype(int)
+df['is_rea'] = (df['round_cat_v2'] == 'REA').astype(int)
+df['is_early'] = df['round_cat_v2'].isin(['ED1', 'ED2', 'EA', 'REA']).astype(int)
+df['round_cat'] = df['round_cat_v2']
 
 # ============================================================
-# 3. PARSE LLM FEATURES (same as bare model)
+# 3. PARSE LLM FEATURES
 # ============================================================
 act_scores = {}
 raw = llm_features_loaded.get('act_scores', {})
@@ -112,11 +253,19 @@ if isinstance(raw, list):
             sid = str(item.get('student_id', '')).replace('.0', '')
             school = str(item.get('school', ''))
             key = f"{sid}_{school}"
+            # FIX: Filter out score=1 entries (no supplement, not real scores)
+            oq = item.get('overall_quality', 0)
+            if isinstance(oq, (int, float)) and oq <= 1:
+                continue  # Skip fake scores
             supp_scores[key] = item
 elif isinstance(raw, dict):
     for key, scores in raw.items():
         if isinstance(scores, dict):
+            oq = scores.get('overall_quality', 0)
+            if isinstance(oq, (int, float)) and oq <= 1:
+                continue  # Skip fake scores
             supp_scores[key] = scores
+print(f"  Supp scores after filtering score=1: {len(supp_scores)} valid entries")
 
 major_diff = llm_features_loaded.get('major_diff', {})
 if isinstance(major_diff, list):
@@ -153,58 +302,62 @@ if not PS_DIMS:
                'coherence_focus', 'overall_effectiveness']
 
 # ============================================================
-# 4. DEFINE STUDENT-LEVEL FEATURE COLUMNS (for residualization)
+# 4. DEFINE FEATURE GROUPS (adapted for v6 = v5 + restored v4 cols)
 # ============================================================
-# These are features that are constant across all applications of the same student
 STUDENT_LEVEL_NUMERIC = [
-    # Test scores
     'toefl', 'sat', 'gpa',
-    # Activity type counts
-    *[f'act_type_count_{t}' for t in [
-        'Academic', 'Art', 'Athletics_Club', 'Athletics_JV', 'Athletics_Varsity',
-        'Career_Oriented', 'Community_Service_Volunteer', 'Computer_Technology',
-        'Cultural', 'Dance', 'Debate_Speech', 'Environmental', 'Family_Responsibilities',
-        'Foreign_Exchange', 'Internship', 'Journalism_Publication',
-        'Junior_ROTC', 'LGBT', 'Music_Instrumental', 'Music_Vocal',
-        'Religious', 'Research', 'Robotics', 'School_Spirit',
-        'Science_Math', 'Social_Justice', 'Student_Govt', 'Theater_Drama', 'Work_Paid'
-    ]],
     'act_total_count', 'act_type_diversity',
-    # Activity PCA
     *[f'act_slot_pca_{i}' for i in range(20)],
-    # Activity BERT PCA
     *[f'act_bert_pca_{i}' for i in range(16)],
-    # PS BERT PCA
+    # Restored from v4: ps_bert_pca
     *[f'ps_bert_pca_{i}' for i in range(16)],
-    # Honors
     'honors_max_score', 'honors_avg_score', 'honors_min_score',
     'honors_count', 'honors_total_score',
-    'honors_has_top_tier', 'honors_tier1_count', 'honors_tier2_count', 'honors_tier3_count',
-    # Summer programs
-    'summer_max_geili', 'summer_avg_geili', 'summer_total_geili',
-    'summer_count', 'summer_has_top_tier', 'summer_top_tier_count',
-    'summer_max_difficulty', 'summer_avg_difficulty',
-    'summer_max_recommendation', 'summer_avg_recommendation',
-    # Cuilu (hs-level, constant per student)
-    'cuilu_hs_top10_rate', 'cuilu_hs_top20_rate', 'cuilu_hs_top30_rate',
-    'cuilu_hs_total_admits', 'cuilu_hs_total_apps',
+    'honors_has_top_tier', 'honors_tier1_count', 'honors_tier2_count',
+    'honors_has_national',
+    # NEW: honors quality ratio
+    'honors_quality_ratio',
+    'cuilu_hs_top10_rate', 'cuilu_hs_top20_rate',
+    'cuilu_hs_top10_count', 'cuilu_hs_top20_count',
+    'cuilu_hs_total',
     'cuilu_feeder_rank', 'cuilu_hs_type_rate', 'cuilu_region_rate',
+    # Restored from v4: hs_to_univ_hist
+    'hs_to_univ_hist_rate', 'hs_to_univ_hist_rate_smoothed', 'hs_to_univ_hist_admits',
+    'hs_overall_hist_rate',
+    # Restored from v4: summer features
+    'summer_max_geili', 'summer_has_elite', 'summer_count',
+    'summer_program_count', 'summer_difficulty_max',
 ]
 
-# Key student features for explicit interactions with school
+# Add act_type_count columns dynamically
+act_type_cols_in_data = [c for c in df.columns if c.startswith('act_type_count_')]
+STUDENT_LEVEL_NUMERIC.extend(act_type_cols_in_data)
+
+# Filter to only existing columns
+STUDENT_LEVEL_NUMERIC = [c for c in STUDENT_LEVEL_NUMERIC if c in df.columns]
+print(f"\n  Student-level numeric features: {len(STUDENT_LEVEL_NUMERIC)}")
+
 KEY_STUDENT_FEATURES = [
     'toefl', 'sat', 'gpa',
     'honors_max_score', 'honors_avg_score', 'honors_count',
-    'summer_max_geili', 'summer_count',
+    'honors_quality_ratio',
     'act_type_diversity', 'act_total_count',
+    # Restored high-value features
+    'hs_to_univ_hist_rate_smoothed',
+    'summer_max_geili',
+]
+
+LLM_INTERACTION_FEATURES = [
+    'llm_act_mean', 'llm_act_max', 'llm_act_avg_power_index',
+    'supp_mean', 'supp_max', 'ps_mean',
+    'major_difficulty',
 ]
 
 # ============================================================
-# 5. BUILD FEATURES (with residualization)
+# 5. BUILD FEATURES
 # ============================================================
 def build_features_base(df):
-    """Build base features WITHOUT residualization (same as bare model).
-    Residualization is done separately inside CV loop to avoid leakage."""
+    """Build base features WITHOUT residualization."""
     df = df.copy()
     
     df['is_partial_year'] = (df['year'] == 2025).astype(int)
@@ -215,57 +368,57 @@ def build_features_base(df):
     for dim in ACT_DIMS:
         col_name = f'llm_act_{dim}'
         df[col_name] = df['sid_str'].map(
-            lambda s, d=dim: safe_num(act_scores.get(s, {}).get(d, -1)))
+            lambda s, d=dim: safe_num(act_scores.get(s, {}).get(d, np.nan)))
     
-    # LLM Supp features (student × school level)
+    # LLM Supp features
     def get_supp_score(row, dim):
         key = f"{row['sid_str']}_{row['school']}"
-        return safe_num(supp_scores.get(key, {}).get(dim, -1))
+        return safe_num(supp_scores.get(key, {}).get(dim, np.nan))
     for dim in SUPP_DIMS:
         col_name = f'supp_{dim}'
         df[col_name] = df.apply(lambda r, d=dim: get_supp_score(r, d), axis=1)
     
-    # Major difficulty (student × school level)
+    # Major difficulty
     def get_major_diff(row):
         key = f"{row['school']}_{row['major_cat']}"
-        return safe_num(major_diff.get(key, {}).get('difficulty_score', -1))
+        return safe_num(major_diff.get(key, {}).get('difficulty_score', np.nan))
     df['major_difficulty'] = df.apply(get_major_diff, axis=1)
     
     # PS Yale scores
     for dim in PS_DIMS:
         col_name = f'ps_{dim}'
         df[col_name] = df['sid_str'].map(
-            lambda s, d=dim: safe_num(ps_yale.get(s, {}).get(d, -1)))
+            lambda s, d=dim: safe_num(ps_yale.get(s, {}).get(d, np.nan)))
     
-    # Aggregate LLM features
+    # Aggregates (use NaN-aware operations)
     llm_act_cols = [f'llm_act_{d}' for d in ACT_DIMS]
-    valid_act = df[llm_act_cols].replace(-1, np.nan)
+    valid_act = df[llm_act_cols]
     df['llm_act_mean'] = valid_act.mean(axis=1)
     df['llm_act_max'] = valid_act.max(axis=1)
     df['llm_act_n_valid'] = valid_act.notna().sum(axis=1)
     
     supp_num_cols = [f'supp_{d}' for d in SUPP_DIMS if d not in ['has_red_flag']]
-    valid_supp = df[supp_num_cols].replace(-1, np.nan)
+    valid_supp = df[supp_num_cols]
     df['supp_mean'] = valid_supp.mean(axis=1)
     df['supp_max'] = valid_supp.max(axis=1)
     
     ps_cols = [f'ps_{d}' for d in PS_DIMS]
-    valid_ps = df[ps_cols].replace(-1, np.nan)
+    valid_ps = df[ps_cols]
     df['ps_mean'] = valid_ps.mean(axis=1)
     
-    # Basic interactions (non-residualized)
+    # Basic interactions (NaN-safe: NaN * anything = NaN, which is fine)
     df['toefl_x_sat'] = df['toefl'] * df['sat'] / 10000.0
     df['gpa_x_toefl'] = df['gpa'] * df['toefl'] / 100.0
-    df['llm_act_x_supp'] = df['llm_act_mean'].fillna(0) * df['supp_mean'].fillna(0)
+    df['llm_act_x_supp'] = df['llm_act_mean'] * df['supp_mean']
     
     if 'honors_avg_score' in df.columns:
-        df['honors_x_sat'] = df['honors_avg_score'].fillna(0) * df['sat'].fillna(0) / 1600
-        df['honors_x_toefl'] = df['honors_avg_score'].fillna(0) * df['toefl'].fillna(0) / 120
+        df['honors_x_sat'] = df['honors_avg_score'] * df['sat'] / 1600
+        df['honors_x_toefl'] = df['honors_avg_score'] * df['toefl'] / 120
     
     if 'cuilu_hs_top10_rate' in df.columns and 'taste_score_sensitivity' in df.columns:
-        df['cuilu_x_taste'] = df['cuilu_hs_top10_rate'].fillna(0) * df['taste_score_sensitivity'].fillna(0)
+        df['cuilu_x_taste'] = df['cuilu_hs_top10_rate'] * df['taste_score_sensitivity']
     
-    # Categoricals
+    # Categoricals - with round_cat v2 (EA/ED1/ED2/REA/RD)
     cat_cols = ['school', 'round_cat', 'major_cat', 'hs_cat', 'year_cat', 'hs_name', 'province']
     cat_cols = [c for c in cat_cols if c in df.columns]
     
@@ -286,86 +439,155 @@ def build_features_base(df):
     return df, cat_cols
 
 
-def add_residualized_features(df, train_mask, cat_cols):
-    """Add residualized + interaction features using ONLY training data statistics.
-    This must be called inside each CV fold to prevent leakage."""
+def add_residualized_features(df, train_mask, cat_cols, selected_features=None):
+    """Add residualized + interaction + ED boost features using ONLY training data statistics.
+    KEY FIX: Residualization uses only non-NaN values for school means."""
     df = df.copy()
     
-    # Step 1: Compute school_base_rate from training data only
+    # Step 1: Bayesian-smoothed school_base_rate
     train_df = df[train_mask]
+    global_rate = train_df[TARGET].mean()
+    
     school_stats = train_df.groupby('school').agg(
-        school_base_rate=(TARGET, 'mean'),
+        school_raw_rate=(TARGET, 'mean'),
         school_n_apps=(TARGET, 'count'),
         school_n_admits=(TARGET, 'sum'),
     ).reset_index()
     
-    # Merge school stats to all rows
-    df = df.merge(school_stats, on='school', how='left')
-    # Fill missing school stats with global mean
-    global_rate = train_df[TARGET].mean()
+    SMOOTH_STRENGTH = 30
+    school_stats['school_base_rate'] = (
+        (school_stats['school_raw_rate'] * school_stats['school_n_apps'] + global_rate * SMOOTH_STRENGTH) /
+        (school_stats['school_n_apps'] + SMOOTH_STRENGTH)
+    )
+    
+    df = df.merge(school_stats[['school', 'school_base_rate', 'school_n_apps', 'school_n_admits']], 
+                  on='school', how='left')
     df['school_base_rate'] = df['school_base_rate'].fillna(global_rate)
     df['school_n_apps'] = df['school_n_apps'].fillna(0)
     df['school_n_admits'] = df['school_n_admits'].fillna(0)
     
-    # Step 2: Compute school-level means for student features (from training data)
-    student_feat_available = [c for c in STUDENT_LEVEL_NUMERIC if c in df.columns]
+    # Step 1b: ED boost per school
+    ed1_mask = train_df['is_ed1'] == 1
+    rd_mask = train_df['is_early'] == 0
     
-    school_means = train_df.groupby('school')[student_feat_available].mean()
+    ed1_school_rates = train_df[ed1_mask].groupby('school')[TARGET].mean()
+    rd_school_rates = train_df[rd_mask].groupby('school')[TARGET].mean()
+    
+    ed_boost_map = {}
+    for school in ed1_school_rates.index:
+        if school in rd_school_rates.index:
+            ed_boost_map[school] = ed1_school_rates[school] - rd_school_rates[school]
+    df['school_ed_boost'] = df['school'].map(ed_boost_map).fillna(0)
+    
+    ed2_mask = train_df['is_ed2'] == 1
+    ed2_school_rates = train_df[ed2_mask].groupby('school')[TARGET].mean()
+    ed2_boost_map = {}
+    for school in ed2_school_rates.index:
+        if school in rd_school_rates.index:
+            ed2_boost_map[school] = ed2_school_rates[school] - rd_school_rates[school]
+    df['school_ed2_boost'] = df['school'].map(ed2_boost_map).fillna(0)
+    
+    # Step 2: Residualize student features
+    # KEY FIX: Use only non-NaN values for school means
+    student_feat_available = [c for c in STUDENT_LEVEL_NUMERIC if c in df.columns]
     
-    # Step 3: Residualize student features
     resid_cols = []
     for col in student_feat_available:
         resid_col = f'{col}_resid'
-        col_school_mean = df['school'].map(school_means[col]).fillna(0)
-        df[resid_col] = df[col].fillna(0) - col_school_mean
+        # Compute school mean using ONLY non-NaN training values
+        school_mean_series = train_df.groupby('school')[col].mean()  # NaN excluded by default
+        col_school_mean = df['school'].map(school_mean_series)
+        # Residual: student value - school mean (NaN if either is NaN)
+        df[resid_col] = df[col] - col_school_mean
         resid_cols.append(resid_col)
     
-    # Step 4: Explicit interactions (student feature × school_base_rate)
+    # Step 3: Explicit interactions (student feature x school_base_rate)
     interaction_cols = []
     for col in KEY_STUDENT_FEATURES:
         if col in df.columns:
             int_col = f'{col}_x_school_rate'
-            df[int_col] = df[col].fillna(0) * df['school_base_rate']
+            df[int_col] = df[col] * df['school_base_rate']  # NaN propagates naturally
             interaction_cols.append(int_col)
             
-            # Also: residualized × school_rate
             resid_col = f'{col}_resid'
             if resid_col in df.columns:
                 int_resid_col = f'{col}_resid_x_rate'
                 df[int_resid_col] = df[resid_col] * df['school_base_rate']
                 interaction_cols.append(int_resid_col)
     
-    # Step 5: Student percentile within school (from training data)
+    # Step 3b: LLM feature x school_base_rate interactions
+    for col in LLM_INTERACTION_FEATURES:
+        if col in df.columns:
+            int_col = f'{col}_x_school_rate'
+            df[int_col] = df[col] * df['school_base_rate']
+            interaction_cols.append(int_col)
+    
+    # Step 3c: portfolio_size x school_base_rate interaction
+    if 'portfolio_size' in df.columns:
+        df['portfolio_x_school_rate'] = df['portfolio_size'] * df['school_base_rate']
+        interaction_cols.append('portfolio_x_school_rate')
+    
+    # Step 3d: ED flag x school_ed_boost interaction
+    if 'is_ed1' in df.columns:
+        df['ed1_x_ed_boost'] = df['is_ed1'] * df['school_ed_boost']
+        interaction_cols.append('ed1_x_ed_boost')
+    if 'is_ed2' in df.columns:
+        df['ed2_x_ed2_boost'] = df['is_ed2'] * df['school_ed2_boost']
+        interaction_cols.append('ed2_x_ed2_boost')
+    
+    # Step 3e: has_sat/has_toefl/has_gpa interactions with school_base_rate
+    for flag in ['has_sat', 'has_toefl', 'has_gpa']:
+        if flag in df.columns:
+            int_col = f'{flag}_x_school_rate'
+            df[int_col] = df[flag] * df['school_base_rate']
+            interaction_cols.append(int_col)
+    
+    # Step 4: Student percentile within school (NaN-safe)
     pctile_cols = []
-    for col in ['toefl', 'sat', 'gpa', 'honors_max_score', 'summer_max_geili']:
+    for col in ['toefl', 'sat', 'gpa', 'honors_max_score',
+                'llm_act_mean', 'supp_mean']:
         if col not in df.columns:
             continue
         pctile_col = f'{col}_school_pctile'
-        # For each school, compute percentile rank using training data distribution
         school_distributions = {}
         for school_id in train_df['school'].unique():
             vals = train_df[train_df['school'] == school_id][col].dropna().values
             if len(vals) > 2:
                 school_distributions[school_id] = vals
         
-        def compute_pctile(row):
+        def compute_pctile(row, col=col, sd=school_distributions):
             school_id = row['school']
             val = row[col]
-            if pd.isna(val) or school_id not in school_distributions:
-                return 0.5  # default to median
-            dist = school_distributions[school_id]
+            if pd.isna(val) or school_id not in sd:
+                return np.nan  # Return NaN instead of 0.5
+            dist = sd[school_id]
             return np.mean(dist <= val)
         
         df[pctile_col] = df.apply(compute_pctile, axis=1)
         pctile_cols.append(pctile_col)
     
+    # Step 5: Student competitiveness score (NaN-safe)
+    if all(c in df.columns for c in ['toefl', 'sat', 'honors_max_score']):
+        # Use NaN-safe computation
+        components = []
+        weights = []
+        for col, w, scale in [('toefl', 0.3, 120), ('sat', 0.3, 1600), 
+                                ('honors_max_score', 0.2, 10), ('llm_act_mean', 0.2, 10)]:
+            if col in df.columns:
+                components.append(df[col] / scale)
+                weights.append(w)
+        if components:
+            strength_df = pd.DataFrame(components).T
+            df['student_strength'] = strength_df.mean(axis=1)  # NaN-safe mean
+            df['strength_vs_school'] = df['student_strength'] - (1 - df['school_base_rate'])
+    
     # Build final feature list
     num_cols = [c for c in df.columns if df[c].dtype in ['float64', 'int64', 'float32', 'int32']
                 and c not in [TARGET, 'student_id', 'year', 'Unnamed: 0']]
     
     all_feat = list(set(num_cols + cat_cols))
     feature_cols = list(dict.fromkeys([c for c in all_feat if c in df.columns]))
-    for remove in [TARGET, 'student_id', 'year', 'sid_str', 'Unnamed: 0']:
+    for remove in [TARGET, 'student_id', 'year', 'sid_str', 'Unnamed: 0', 'portfolio_size_raw']:
         if remove in feature_cols:
             feature_cols.remove(remove)
     
@@ -373,18 +595,28 @@ def add_residualized_features(df, train_mask, cat_cols):
     to_drop = [c for c in feature_cols if df[c].nunique() <= 1]
     feature_cols = [c for c in feature_cols if c not in to_drop]
     
-    # Fill NaN
-    for c in feature_cols:
-        if df[c].isnull().any():
-            df[c] = df[c].fillna(-1)
+    # Apply feature selection if provided
+    if selected_features is not None:
+        must_keep = set(cat_cols) | {'school_base_rate', 'school_n_apps', 'school_n_admits',
+                                      'student_strength', 'strength_vs_school',
+                                      'school_ed_boost', 'school_ed2_boost',
+                                      'is_ed1', 'is_ed2', 'is_rea', 'is_early',
+                                      'ed1_x_ed_boost', 'ed2_x_ed2_boost',
+                                      'has_sat', 'has_toefl', 'has_gpa',
+                                      'portfolio_size', 'portfolio_size_bin', 'portfolio_x_school_rate'}
+        feature_cols = [c for c in feature_cols if c in selected_features or c in must_keep]
+    
+    # KEY CHANGE: Do NOT fill NaN for CatBoost - it handles NaN natively
+    # Only fill NaN for LGB/XGB later, and only for non-cat columns
+    # For now, just handle inf
     for c in feature_cols:
         if df[c].dtype in ['float64', 'float32']:
-            df[c] = df[c].replace([np.inf, -np.inf], -1)
+            df[c] = df[c].replace([np.inf, -np.inf], np.nan)
     
     cat_indices = [feature_cols.index(c) for c in cat_cols if c in feature_cols]
     
-    new_feat_count = len(resid_cols) + len(interaction_cols) + len(pctile_cols) + 3  # +3 for school stats
-    print(f"    Residualized features added: {len(resid_cols)} resid + {len(interaction_cols)} interactions + {len(pctile_cols)} percentiles + 3 school stats = {new_feat_count} new")
+    new_feat_count = len(resid_cols) + len(interaction_cols) + len(pctile_cols) + 5
+    print(f"    Resid features: {len(resid_cols)} resid + {len(interaction_cols)} interact + {len(pctile_cols)} pctile = {new_feat_count} new, total={len(feature_cols)}")
     
     return df, feature_cols, cat_cols, cat_indices
 
@@ -395,14 +627,91 @@ def add_residualized_features(df, train_mask, cat_cols):
 df_base, cat_cols = build_features_base(df)
 print(f"\nBase features built. Shape: {df_base.shape}")
 
+# Quick NaN summary
+print(f"\n  NaN summary after fixes:")
+for col in ['sat', 'toefl', 'gpa']:
+    nan_pct = df_base[col].isna().mean() * 100
+    print(f"    {col}: {nan_pct:.1f}% NaN")
+
 y = df_base[TARGET].values
 groups = df_base['student_id'].values
 
 # ============================================================
-# 7. TEMPORAL VALIDATION (2020-2023 → 2024) WITH RESIDUALIZATION
+# 7. STAGE 1: FEATURE IMPORTANCE ESTIMATION
+# ============================================================
+print(f"\n{'='*70}")
+print(f"  STAGE 1: FEATURE IMPORTANCE ESTIMATION")
+print(f"{'='*70}")
+
+stage1_fi = []
+gkf_s1 = GroupKFold(n_splits=5)
+for fold, (tr_idx, va_idx) in enumerate(gkf_s1.split(df_base, y, groups)):
+    train_mask = pd.Series(False, index=df_base.index)
+    train_mask.iloc[tr_idx] = True
+    
+    df_fold, feat_cols_f, cat_cols_f, cat_idx_f = add_residualized_features(
+        df_base, train_mask, cat_cols)
+    
+    X_tr = df_fold[feat_cols_f].iloc[tr_idx]
+    X_va = df_fold[feat_cols_f].iloc[va_idx]
+    y_tr = y[tr_idx]
+    y_va = y[va_idx]
+    
+    for c in cat_cols_f:
+        if c in X_tr.columns:
+            X_tr[c] = X_tr[c].astype(int)
+            X_va[c] = X_va[c].astype(int)
+    # CatBoost handles NaN natively - don't fill
+    
+    cb = CatBoostClassifier(
+        iterations=500, depth=6, learning_rate=0.05,
+        l2_leaf_reg=7, random_seed=42, verbose=0,
+        cat_features=cat_idx_f, eval_metric='AUC',
+        early_stopping_rounds=50)
+    pool_tr = Pool(X_tr, y_tr, cat_features=cat_idx_f)
+    pool_va = Pool(X_va, y_va, cat_features=cat_idx_f)
+    cb.fit(pool_tr, eval_set=pool_va, verbose=0)
+    
+    fi = cb.get_feature_importance()
+    stage1_fi.append(fi)
+    
+    auc = roc_auc_score(y_va, cb.predict_proba(Pool(X_va, cat_features=cat_idx_f))[:, 1])
+    print(f"  Fold {fold+1}/5: AUC={auc:.4f}, Features={len(feat_cols_f)}")
+    
+    if fold == 0:
+        all_feature_names = feat_cols_f
+    
+    del cb, pool_tr, pool_va, df_fold; gc.collect()
+
+# Select top features
+avg_fi = np.mean(stage1_fi, axis=0)
+fi_pairs = sorted(zip(all_feature_names, avg_fi), key=lambda x: -x[1])
+
+selected_set = set(cat_cols)
+n_added = 0
+for fname, imp in fi_pairs:
+    if fname not in cat_cols:
+        selected_set.add(fname)
+        n_added += 1
+        if n_added >= FEATURE_SELECT_TOP_N:
+            break
+
+print(f"\n  Feature selection: {len(all_feature_names)} -> {len(selected_set)} features")
+print(f"  Top 20 features:")
+for i, (fname, imp) in enumerate(fi_pairs[:20]):
+    marker = ""
+    if '_resid' in fname: marker = " [R]"
+    elif '_x_school_rate' in fname or '_resid_x_rate' in fname or '_x_ed' in fname: marker = " [I]"
+    elif '_school_pctile' in fname: marker = " [P]"
+    elif 'school_base_rate' in fname: marker = " [S]"
+    elif 'ed_boost' in fname: marker = " [ED]"
+    print(f"    {i+1:3d}. {fname:<50s} {imp:>8.2f}{marker}")
+
+# ============================================================
+# 8. TEMPORAL VALIDATION WITH SELECTED FEATURES
 # ============================================================
 print(f"\n{'='*70}")
-print(f"  TEMPORAL VALIDATION (2020-2023 → 2024) WITH RESIDUALIZATION")
+print(f"  TEMPORAL VALIDATION (2020-2023 -> 2024) WITH FEATURE SELECTION")
 print(f"{'='*70}")
 
 mask_train_temporal = df_base['year'].isin([2020, 2021, 2022, 2023])
@@ -410,66 +719,72 @@ mask_test_temporal = df_base['year'] == 2024
 
 temporal_results = {}
 if mask_test_temporal.sum() > 0:
-    # Build residualized features using temporal train data
     df_temporal, feat_cols_t, cat_cols_t, cat_idx_t = add_residualized_features(
-        df_base, mask_train_temporal, cat_cols)
+        df_base, mask_train_temporal, cat_cols, selected_features=selected_set)
     
     X_t = df_temporal[feat_cols_t].copy()
     for c in cat_cols_t:
         if c in X_t.columns:
             X_t[c] = X_t[c].astype(int)
-    X_t = X_t.fillna(-1)
     
     X_tr_t = X_t[mask_train_temporal]
     X_te_t = X_t[mask_test_temporal]
     y_tr_t = y[mask_train_temporal]
     y_te_t = y[mask_test_temporal]
     
+    # For LGB/XGB: fill NaN with -999 (a value they can split on)
+    X_tr_t_filled = X_tr_t.fillna(-999)
+    X_te_t_filled = X_te_t.fillna(-999)
+    
     print(f"  Train: {len(X_tr_t)}, Test: {len(X_te_t)}, Features: {len(feat_cols_t)}")
     
     for seed in SEEDS:
+        # CatBoost: native NaN handling
         cb_t = CatBoostClassifier(
-            iterations=1000, depth=6, learning_rate=0.05,
-            l2_leaf_reg=5, random_seed=seed, verbose=0,
+            iterations=1000, depth=6, learning_rate=0.03,
+            l2_leaf_reg=7, random_seed=seed, verbose=0,
             cat_features=cat_idx_t, eval_metric='AUC',
-            early_stopping_rounds=80)
+            early_stopping_rounds=100, min_data_in_leaf=10)
         pool_tr = Pool(X_tr_t, y_tr_t, cat_features=cat_idx_t)
         pool_te = Pool(X_te_t, y_te_t, cat_features=cat_idx_t)
         cb_t.fit(pool_tr, eval_set=pool_te, verbose=0)
         cb_pred = cb_t.predict_proba(Pool(X_te_t, cat_features=cat_idx_t))[:, 1]
         del cb_t; gc.collect()
         
-        lgb_tr = lgb.Dataset(X_tr_t.values, y_tr_t, categorical_feature=cat_idx_t)
-        lgb_va = lgb.Dataset(X_te_t.values, y_te_t, categorical_feature=cat_idx_t, reference=lgb_tr)
+        # LGB: use filled data
+        lgb_tr = lgb.Dataset(X_tr_t_filled.values, y_tr_t, categorical_feature=cat_idx_t)
+        lgb_va = lgb.Dataset(X_te_t_filled.values, y_te_t, categorical_feature=cat_idx_t, reference=lgb_tr)
         lgb_params = {
             'objective': 'binary', 'metric': 'auc', 'verbosity': -1,
-            'learning_rate': 0.05, 'num_leaves': 63, 'max_depth': 6,
-            'min_child_samples': 20, 'reg_alpha': 0.1, 'reg_lambda': 1.0,
-            'feature_fraction': 0.8, 'bagging_fraction': 0.8, 'bagging_freq': 5,
+            'learning_rate': 0.03, 'num_leaves': 63, 'max_depth': 6,
+            'min_child_samples': 25, 'reg_alpha': 0.3, 'reg_lambda': 2.0,
+            'feature_fraction': 0.7, 'bagging_fraction': 0.8, 'bagging_freq': 5,
             'seed': seed
         }
-        lgb_model = lgb.train(lgb_params, lgb_tr, num_boost_round=1000,
+        lgb_model = lgb.train(lgb_params, lgb_tr, num_boost_round=1500,
                               valid_sets=[lgb_va],
-                              callbacks=[lgb.early_stopping(80), lgb.log_evaluation(0)])
-        lgb_pred = lgb_model.predict(X_te_t.values)
+                              callbacks=[lgb.early_stopping(100), lgb.log_evaluation(0)])
+        lgb_pred = lgb_model.predict(X_te_t_filled.values)
         del lgb_model; gc.collect()
         
-        dtrain = xgb.DMatrix(X_tr_t.values, label=y_tr_t, enable_categorical=False)
-        dtest = xgb.DMatrix(X_te_t.values, label=y_te_t, enable_categorical=False)
+        # XGB: use filled data
+        dtrain = xgb.DMatrix(X_tr_t_filled.values, label=y_tr_t, enable_categorical=False)
+        dtest = xgb.DMatrix(X_te_t_filled.values, label=y_te_t, enable_categorical=False)
         xgb_params = {
             'objective': 'binary:logistic', 'eval_metric': 'auc',
-            'max_depth': 6, 'learning_rate': 0.05,
-            'subsample': 0.8, 'colsample_bytree': 0.8,
-            'reg_alpha': 0.1, 'reg_lambda': 1.0,
+            'max_depth': 6, 'learning_rate': 0.03,
+            'subsample': 0.8, 'colsample_bytree': 0.7,
+            'reg_alpha': 0.3, 'reg_lambda': 2.0,
+            'min_child_weight': 5,
             'seed': seed, 'verbosity': 0
         }
-        xgb_model = xgb.train(xgb_params, dtrain, num_boost_round=1000,
+        xgb_model = xgb.train(xgb_params, dtrain, num_boost_round=1500,
                               evals=[(dtest, 'val')],
-                              early_stopping_rounds=80, verbose_eval=False)
+                              early_stopping_rounds=100, verbose_eval=False)
         xgb_pred = xgb_model.predict(dtest)
         del xgb_model, dtrain, dtest; gc.collect()
         
-        blend = 0.4 * cb_pred + 0.3 * lgb_pred + 0.3 * xgb_pred
+        blend = 0.45 * cb_pred + 0.20 * lgb_pred + 0.35 * xgb_pred
         temporal_results[seed] = {
             'cb': float(roc_auc_score(y_te_t, cb_pred)),
             'lgb': float(roc_auc_score(y_te_t, lgb_pred)),
@@ -479,19 +794,19 @@ if mask_test_temporal.sum() > 0:
         print(f"  Seed {seed}: CB={temporal_results[seed]['cb']:.4f} LGB={temporal_results[seed]['lgb']:.4f} XGB={temporal_results[seed]['xgb']:.4f} Blend={temporal_results[seed]['blend']:.4f}")
     
     avg_temporal = np.mean([v['blend'] for v in temporal_results.values()])
-    print(f"\n  AVG Temporal Blend: {avg_temporal:.4f} (V37.3: 0.8410, V38.2-bare: 0.8417)")
+    print(f"\n  AVG Temporal Blend: {avg_temporal:.4f} (V37.3: 0.8410, V38.2-PRO-V3: 0.8528)")
     print(f"  Delta vs V37.3: {avg_temporal - 0.8410:+.4f}")
-    print(f"  Delta vs V38.2-bare: {avg_temporal - 0.8417:+.4f}")
+    print(f"  Delta vs V38.2-PRO-V3: {avg_temporal - 0.8528:+.4f}")
     
     del df_temporal, X_t; gc.collect()
 else:
     avg_temporal = 0.0
 
 # ============================================================
-# 8. MULTI-SEED GROUPKFOLD WITH RESIDUALIZATION INSIDE FOLDS
+# 9. STAGE 2: MULTI-SEED GROUPKFOLD
 # ============================================================
 print(f"\n{'='*70}")
-print(f"  MULTI-SEED GROUPKFOLD WITH RESIDUALIZATION ({len(SEEDS)} seeds x {N_FOLDS} folds)")
+print(f"  STAGE 2: MULTI-SEED GROUPKFOLD ({len(SEEDS)} seeds x {N_FOLDS} folds)")
 print(f"{'='*70}")
 
 all_cb_oof = []
@@ -508,35 +823,32 @@ for seed_idx, seed in enumerate(SEEDS):
     xgb_oof = np.zeros(len(df_base))
     
     for fold, (tr_idx, va_idx) in enumerate(gkf.split(df_base, y, groups)):
-        # Create train mask for residualization
         train_mask = pd.Series(False, index=df_base.index)
         train_mask.iloc[tr_idx] = True
         
-        # Build residualized features using ONLY training fold data
         df_fold, feat_cols_f, cat_cols_f, cat_idx_f = add_residualized_features(
-            df_base, train_mask, cat_cols)
+            df_base, train_mask, cat_cols, selected_features=selected_set)
         
         if feature_cols_final is None:
             feature_cols_final = feat_cols_f
-            print(f"    Total features with residualization: {len(feat_cols_f)}")
+            print(f"    Total features after selection: {len(feat_cols_f)}")
         
         X_fold = df_fold[feat_cols_f].copy()
         for c in cat_cols_f:
             if c in X_fold.columns:
                 X_fold[c] = X_fold[c].astype(int)
-        X_fold = X_fold.fillna(-1)
         
         X_tr_df = X_fold.iloc[tr_idx]
         X_va_df = X_fold.iloc[va_idx]
         y_tr = y[tr_idx]
         y_va = y[va_idx]
         
-        # CatBoost
+        # CatBoost: native NaN
         cb = CatBoostClassifier(
-            iterations=1000, depth=6, learning_rate=0.05,
-            l2_leaf_reg=5, random_seed=seed, verbose=0,
+            iterations=1500, depth=6, learning_rate=0.03,
+            l2_leaf_reg=7, random_seed=seed, verbose=0,
             cat_features=cat_idx_f, eval_metric='AUC',
-            early_stopping_rounds=80)
+            early_stopping_rounds=100, min_data_in_leaf=10)
         pool_tr = Pool(X_tr_df, y_tr, cat_features=cat_idx_f)
         pool_va = Pool(X_va_df, y_va, cat_features=cat_idx_f)
         cb.fit(pool_tr, eval_set=pool_va, verbose=0)
@@ -547,37 +859,39 @@ for seed_idx, seed in enumerate(SEEDS):
             all_fi.append(cb.get_feature_importance())
         del cb, pool_tr, pool_va; gc.collect()
         
-        # LightGBM
-        X_tr_arr, X_va_arr = X_tr_df.values, X_va_df.values
-        lgb_tr = lgb.Dataset(X_tr_arr, y_tr, categorical_feature=cat_idx_f)
-        lgb_va_ds = lgb.Dataset(X_va_arr, y_va, categorical_feature=cat_idx_f, reference=lgb_tr)
+        # LGB/XGB: fill NaN
+        X_tr_filled = X_tr_df.fillna(-999).values
+        X_va_filled = X_va_df.fillna(-999).values
+        
+        lgb_tr = lgb.Dataset(X_tr_filled, y_tr, categorical_feature=cat_idx_f)
+        lgb_va_ds = lgb.Dataset(X_va_filled, y_va, categorical_feature=cat_idx_f, reference=lgb_tr)
         lgb_params = {
             'objective': 'binary', 'metric': 'auc', 'verbosity': -1,
-            'learning_rate': 0.05, 'num_leaves': 63, 'max_depth': 6,
-            'min_child_samples': 20, 'reg_alpha': 0.1, 'reg_lambda': 1.0,
-            'feature_fraction': 0.8, 'bagging_fraction': 0.8, 'bagging_freq': 5,
+            'learning_rate': 0.03, 'num_leaves': 63, 'max_depth': 6,
+            'min_child_samples': 25, 'reg_alpha': 0.3, 'reg_lambda': 2.0,
+            'feature_fraction': 0.7, 'bagging_fraction': 0.8, 'bagging_freq': 5,
             'seed': seed
         }
-        lgb_model = lgb.train(lgb_params, lgb_tr, num_boost_round=1000,
+        lgb_model = lgb.train(lgb_params, lgb_tr, num_boost_round=1500,
                               valid_sets=[lgb_va_ds],
-                              callbacks=[lgb.early_stopping(80), lgb.log_evaluation(0)])
-        lgb_pred = lgb_model.predict(X_va_arr)
+                              callbacks=[lgb.early_stopping(100), lgb.log_evaluation(0)])
+        lgb_pred = lgb_model.predict(X_va_filled)
         lgb_oof[va_idx] = lgb_pred
         del lgb_model; gc.collect()
         
-        # XGBoost
-        dtrain = xgb.DMatrix(X_tr_arr, label=y_tr)
-        dval = xgb.DMatrix(X_va_arr, label=y_va)
+        dtrain = xgb.DMatrix(X_tr_filled, label=y_tr)
+        dval = xgb.DMatrix(X_va_filled, label=y_va)
         xgb_params = {
             'objective': 'binary:logistic', 'eval_metric': 'auc',
-            'max_depth': 6, 'learning_rate': 0.05,
-            'subsample': 0.8, 'colsample_bytree': 0.8,
-            'reg_alpha': 0.1, 'reg_lambda': 1.0,
+            'max_depth': 6, 'learning_rate': 0.03,
+            'subsample': 0.8, 'colsample_bytree': 0.7,
+            'reg_alpha': 0.3, 'reg_lambda': 2.0,
+            'min_child_weight': 5,
             'seed': seed, 'verbosity': 0
         }
-        xgb_model = xgb.train(xgb_params, dtrain, num_boost_round=1000,
+        xgb_model = xgb.train(xgb_params, dtrain, num_boost_round=1500,
                               evals=[(dval, 'val')],
-                              early_stopping_rounds=80, verbose_eval=False)
+                              early_stopping_rounds=100, verbose_eval=False)
         xgb_pred = xgb_model.predict(dval)
         xgb_oof[va_idx] = xgb_pred
         del xgb_model, dtrain, dval, df_fold, X_fold; gc.collect()
@@ -595,7 +909,7 @@ for seed_idx, seed in enumerate(SEEDS):
     all_xgb_oof.append(xgb_oof)
 
 # ============================================================
-# 9. ENSEMBLE & BLEND
+# 10. ENSEMBLE & BLEND
 # ============================================================
 print(f"\n{'='*70}")
 print(f"  ENSEMBLE RESULTS")
@@ -609,14 +923,14 @@ cb_final_auc = roc_auc_score(y, cb_avg)
 lgb_final_auc = roc_auc_score(y, lgb_avg)
 xgb_final_auc = roc_auc_score(y, xgb_avg)
 
-print(f"  CB  5-seed avg: {cb_final_auc:.4f}")
-print(f"  LGB 5-seed avg: {lgb_final_auc:.4f}")
-print(f"  XGB 5-seed avg: {xgb_final_auc:.4f}")
+print(f"  CB  {len(SEEDS)}-seed avg: {cb_final_auc:.4f}")
+print(f"  LGB {len(SEEDS)}-seed avg: {lgb_final_auc:.4f}")
+print(f"  XGB {len(SEEDS)}-seed avg: {xgb_final_auc:.4f}")
 
 best_auc = 0
-best_weights = (0.4, 0.3, 0.3)
+best_weights = (0.45, 0.20, 0.35)
 for w_cb in np.arange(0.2, 0.7, 0.05):
-    for w_lgb in np.arange(0.1, 0.6, 0.05):
+    for w_lgb in np.arange(0.05, 0.5, 0.05):
         w_xgb = 1.0 - w_cb - w_lgb
         if w_xgb < 0.05: continue
         blend = w_cb * cb_avg + w_lgb * lgb_avg + w_xgb * xgb_avg
@@ -626,26 +940,26 @@ for w_cb in np.arange(0.2, 0.7, 0.05):
             best_weights = (w_cb, w_lgb, w_xgb)
 
 print(f"\n  Best 3-model blend: {best_auc:.4f}")
-print(f"  Delta vs V37.3:     {best_auc - 0.8697:+.4f}")
-print(f"  Delta vs V38.2-bare: {best_auc - 0.8687:+.4f}")
+print(f"  Delta vs V37.3:       {best_auc - 0.8697:+.4f}")
+print(f"  Delta vs V38.2-PRO-V3: {best_auc - 0.8743:+.4f}")
 print(f"  Weights: CB={best_weights[0]:.2f} LGB={best_weights[1]:.2f} XGB={best_weights[2]:.2f}")
 
 rank_blend = (rankdata(cb_avg) + rankdata(lgb_avg) + rankdata(xgb_avg)) / 3
 rank_auc = roc_auc_score(y, rank_blend)
 print(f"  Rank blend: {rank_auc:.4f}")
 
-final_blend = best_weights[0] * cb_avg + best_weights[1] * lgb_avg + best_weights[2] * xgb_avg
-final_auc = roc_auc_score(y, final_blend)
-final_brier = brier_score_loss(y, final_blend)
-final_logloss = log_loss(y, np.clip(final_blend, 1e-7, 1-1e-7))
+final_blend_prob = best_weights[0] * cb_avg + best_weights[1] * lgb_avg + best_weights[2] * xgb_avg
+final_auc = roc_auc_score(y, final_blend_prob)
+final_brier = brier_score_loss(y, np.clip(final_blend_prob, 1e-7, 1-1e-7))
+final_logloss = log_loss(y, np.clip(final_blend_prob, 1e-7, 1-1e-7))
 
 print(f"\n  FINAL METRICS:")
-print(f"    AUC:      {final_auc:.4f} (V37.3: 0.8697, V38.2-bare: 0.8687)")
+print(f"    AUC:      {final_auc:.4f} (V37.3: 0.8697, V38.2-PRO-V3: 0.8743)")
 print(f"    Brier:    {final_brier:.4f}")
 print(f"    LogLoss:  {final_logloss:.4f}")
 
 # ============================================================
-# 10. FEATURE IMPORTANCE
+# 11. FEATURE IMPORTANCE
 # ============================================================
 print(f"\n{'='*70}")
 print(f"  FEATURE IMPORTANCE (avg across seeds)")
@@ -654,36 +968,46 @@ print(f"{'='*70}")
 if feature_cols_final and all_fi:
     avg_fi = np.mean(all_fi, axis=0)
     fi_pairs = sorted(zip(feature_cols_final, avg_fi), key=lambda x: -x[1])
+    
     print(f"  {'Rank':<5s} {'Feature':<50s} {'Importance':>10s}")
     print(f"  {'-'*5} {'-'*50} {'-'*10}")
     for i, (fname, imp) in enumerate(fi_pairs[:50]):
         marker = ""
-        if '_resid' in fname:
-            marker = " [RESID]"
-        elif '_x_school_rate' in fname or '_resid_x_rate' in fname:
-            marker = " [INTERACT]"
-        elif '_school_pctile' in fname:
-            marker = " [PCTILE]"
-        elif fname.startswith('school_base_rate'):
-            marker = " [SCHOOL_RATE]"
+        if '_resid' in fname: marker = " [RESID]"
+        elif '_x_school_rate' in fname or '_resid_x_rate' in fname or '_x_ed' in fname: marker = " [INTERACT]"
+        elif '_school_pctile' in fname: marker = " [PCTILE]"
+        elif fname.startswith('school_base_rate'): marker = " [SCHOOL_RATE]"
+        elif 'ed_boost' in fname or 'ed2_boost' in fname: marker = " [ED_BOOST]"
+        elif fname.startswith('has_'): marker = " [FLAG]"
         print(f"  {i+1:<5d} {fname:<50s} {imp:>10.2f}{marker}")
     
-    # Count new residualized features in top 30
-    resid_in_top30 = sum(1 for f, _ in fi_pairs[:30] if '_resid' in f or '_x_school_rate' in f or '_school_pctile' in f or 'school_base_rate' in f)
+    resid_in_top30 = sum(1 for f, _ in fi_pairs[:30] 
+                         if '_resid' in f or '_x_school_rate' in f or '_school_pctile' in f or 'school_base_rate' in f)
     print(f"\n  Residualized/interaction features in top 30: {resid_in_top30}")
 
 # ============================================================
-# 11. SAVE RESULTS
+# 12. SAVE RESULTS
 # ============================================================
 elapsed = time.time() - start_time
 
 results = {
-    'version': 'V38.2-residualized',
+    'version': 'V38.2-pro-v4',
     'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'),
     'elapsed_minutes': elapsed / 60,
+    'changes': [
+        'Use v5 feature matrix (GPA fixed: 40.3% coverage)',
+        'SAT=0 -> NaN + has_sat flag',
+        'TOEFL=0 -> NaN + has_toefl flag',
+        'GPA=0 -> NaN + has_gpa flag',
+        '-1 -> NaN for sentinel columns',
+        'Residualization uses only non-NaN values',
+        'CatBoost native NaN handling',
+        'LGB/XGB use -999 for NaN',
+        'Percentile returns NaN instead of 0.5 for missing',
+    ],
     'comparison': {
         'v37_3': {'auc': 0.8697, 'temporal_auc': 0.8410},
-        'v38_2_bare': {'auc': 0.8687, 'temporal_auc': 0.8417},
+        'v38_2_pro_v3': {'auc': 0.8743, 'temporal_auc': 0.8528},
     },
     'temporal_validation': {
         'per_seed': temporal_results,
@@ -703,20 +1027,20 @@ results = {
     'feature_importance': [[f, float(i)] for f, i in fi_pairs[:50]] if feature_cols_final and all_fi else [],
 }
 
-with open(os.path.join(OUTPUT_DIR, 'v38_2_resid_results.json'), 'w') as f:
+with open(os.path.join(OUTPUT_DIR, 'v38_2_pro_v4_results.json'), 'w') as f:
     json.dump(results, f, indent=2)
 
 oof_df = df_base[['student_id', 'school', 'year', TARGET]].copy()
 oof_df['cb_pred'] = cb_avg
 oof_df['lgb_pred'] = lgb_avg
 oof_df['xgb_pred'] = xgb_avg
-oof_df['final_pred'] = final_blend
-oof_df.to_csv(os.path.join(OUTPUT_DIR, 'v38_2_resid_oof_predictions.csv'), index=False)
+oof_df['final_pred'] = final_blend_prob
+oof_df.to_csv(os.path.join(OUTPUT_DIR, 'v38_2_pro_v4_oof_predictions.csv'), index=False)
 
 print(f"\n{'='*70}")
-print(f"  V38.2 RESIDUALIZED MODEL COMPLETE")
+print(f"  V38.2-PRO-V4 COMPLETE")
 print(f"  Total time: {elapsed/60:.1f} minutes")
 print(f"  Features: {len(feature_cols_final) if feature_cols_final else 'N/A'}")
-print(f"  GroupKFold AUC: {final_auc:.4f} (V37.3: 0.8697, V38.2-bare: 0.8687)")
-print(f"  Temporal AUC:   {avg_temporal:.4f} (V37.3: 0.8410, V38.2-bare: 0.8417)")
+print(f"  GroupKFold AUC: {final_auc:.4f} (V37.3: 0.8697, V38.2-PRO-V3: 0.8743)")
+print(f"  Temporal AUC:   {avg_temporal:.4f} (V37.3: 0.8410, V38.2-PRO-V2: 0.8469)")
 print(f"{'='*70}")