v2: analyze_results.py

scripts/analyze_results.py

@@ -61,22 +61,36 @@ def process_synthetic(df):
 
 
 def compute_correlations(df):
-    """Compute correlation table between proxies and error metrics."""
+    """Compute correlation table between proxies and error metrics.
+    
+    Computes within-regime correlations (controlling for graph structure)
+    and also log-transformed chi correlations.
+    """
     rows = []
     
     proxy_cols = ['chi_seed_max', 'chi_seed_sum', 'seed_degree']
+    log_proxy_cols = ['log_chi_max', 'log_chi_sum']
     target_cols = ['rel_error_R2', 'rel_error_R4', 'interference_cosine_R2']
     
-    # By dataset/regime grouping
-    if 'dataset_name' in df.columns:
-        groups = df.groupby('dataset_name')
-    elif 'regime' in df.columns:
-        groups = df.groupby('regime')
+    # Add log-chi columns
+    df_copy = df.copy()
+    if 'chi_seed_max' in df_copy.columns:
+        df_copy['log_chi_max'] = np.log1p(df_copy['chi_seed_max'].clip(lower=0))
+    if 'chi_seed_sum' in df_copy.columns:
+        df_copy['log_chi_sum'] = np.log1p(df_copy['chi_seed_sum'].clip(lower=0))
+    
+    all_proxies = proxy_cols + log_proxy_cols
+    
+    # Within-regime correlations (most informative)
+    if 'regime' in df_copy.columns:
+        regime_groups = df_copy.groupby('regime')
+    elif 'dataset_name' in df_copy.columns:
+        regime_groups = df_copy.groupby('dataset_name')
     else:
-        groups = [('all', df)]
+        regime_groups = [('all', df_copy)]
     
-    for grp_name, grp_df in groups:
-        for proxy in proxy_cols:
+    for grp_name, grp_df in regime_groups:
+        for proxy in all_proxies:
             for target in target_cols:
                 if proxy in grp_df.columns and target in grp_df.columns:
                     x = grp_df[proxy].dropna()
@@ -88,8 +102,13 @@ def compute_correlations(df):
                     x, y = x[mask], y[mask]
                     
                     if len(x) >= 5:
-                        pr, pp = stats.pearsonr(x, y)
-                        sr, sp = stats.spearmanr(x, y)
+                        try:
+                            pr, pp = stats.pearsonr(x, y)
+                            sr, sp = stats.spearmanr(x, y)
+                        except:
+                            continue
+                        if np.isnan(pr) or np.isnan(sr):
+                            continue
                         rows.append({
                             'dataset_regime': grp_name,
                             'model_family': grp_df['model_family'].iloc[0] if 'model_family' in grp_df.columns else 'unknown',
@@ -245,6 +264,35 @@ def main():
             save_table(real_summary, os.path.join(tables_dir, 'table_real_datasets'),
                       'Real Dataset Summary')
     
+    # Bootstrap CIs for key metrics
+    from src.metrics import compute_bootstrap_summary
+    
+    metric_cols = ['empirical_decay_mu', 'rel_error_R1', 'rel_error_R2', 'rel_error_R3', 
+                   'rel_error_R4', 'chi_seed_max', 'interference_cosine_R2',
+                   'rel_error_warm_start', 'rel_error_one_step']
+    
+    if len(syn_df) > 0:
+        boot_syn = compute_bootstrap_summary(
+            syn_df, ['graph_type', 'prior_strength', 'K'], metric_cols)
+        if len(boot_syn) > 0:
+            save_table(boot_syn, os.path.join(tables_dir, 'table_synthetic_bootstrap'),
+                      'Synthetic Bootstrap CIs')
+    
+    if len(real_df) > 0:
+        boot_real = compute_bootstrap_summary(
+            real_df, ['dataset_name', 'K'], metric_cols)
+        if len(boot_real) > 0:
+            save_table(boot_real, os.path.join(tables_dir, 'table_real_bootstrap'),
+                      'Real Data Bootstrap CIs')
+    
+    if 'model_family' in df.columns and df['model_family'].nunique() > 1:
+        boot_mf = compute_bootstrap_summary(
+            df[df['dataset_type'] == 'synthetic'], 
+            ['model_family', 'graph_type'], metric_cols)
+        if len(boot_mf) > 0:
+            save_table(boot_mf, os.path.join(tables_dir, 'table_model_family_bootstrap'),
+                      'Model Family Bootstrap CIs')
+    
     # Table 3: Correlations
     corr_df = compute_correlations(df)
     if len(corr_df) > 0: