Update src/streamlit_app.py

src/streamlit_app.py

@@ -1,3 +1,15 @@
+"""
+COMPOSITE CORRELATION CALCULATOR - COMPLETE EXPLANATION
+========================================================
+
+This module implements unit-weighted composite correlation calculation from a correlation matrix.
+It uses the classical test theory formula to compute correlations between a composite (sum of items)
+and all other variables, without needing raw data.
+
+Author: HubMeta Team
+Date: February 2026
+"""
+
 import os
 # On Huggingface Spaces the home directory may be unwritable; override it to the current working directory
 os.environ['HOME'] = os.getcwd()
@@ -8,35 +20,213 @@ import streamlit as st
 import pandas as pd
 import numpy as np
 
+
 def composite_correlations(R, composite_idx, var_names=None, augment=False):
-    """Compute unit-weighted composite correlations."""
+    """
+    Compute unit-weighted composite correlations from a correlation matrix.
+    
+    This function calculates the correlation between a composite variable (formed by
+    summing multiple items) and all other variables in the correlation matrix. The
+    calculation is based on classical test theory and uses the psychometric formula
+    for composite reliability.
+    
+    Mathematical Background
+    -----------------------
+    For a unit-weighted composite Y = X₁ + X₂ + ... + Xₖ, the correlation between
+    Y and any variable Z is:
+    
+        r(Y, Z) = Σᵢ r(Xᵢ, Z) / σ_Y
+        
+    where σ_Y = sqrt(k + k(k-1)×r̄)
+    
+    Here:
+    - k = number of items in the composite
+    - r̄ = average inter-item correlation (mean of off-diagonal correlations in R_yy)
+    - Σᵢ r(Xᵢ, Z) = sum of correlations between each composite item and variable Z
+    
+    This formula is mathematically equivalent to:
+    1. Creating composite scores by summing items
+    2. Correlating the composite with each variable
+    
+    But it works directly from the correlation matrix without needing raw data.
+    
+    Parameters
+    ----------
+    R : array-like, shape (n_vars, n_vars)
+        Full correlation matrix. Can be a numpy array or pandas DataFrame.
+        Must be symmetric with 1s on the diagonal.
+        
+    composite_idx : list of int
+        Indices of variables to include in the composite.
+        Example: [0, 2, 5] means use variables at positions 0, 2, and 5.
+        
+    var_names : list of str, optional
+        Names of all variables in R. If provided, output will be a labeled Series.
+        Length must match R.shape[0].
+        
+    augment : bool, default=False
+        If True, return both the composite correlations AND an augmented correlation
+        matrix that includes the composite as a new row/column.
+        
+    Returns
+    -------
+    r_comp : array or Series
+        Correlations between the composite and each variable.
+        - If var_names is None: numpy array of shape (n_vars,)
+        - If var_names provided: pandas Series with variable names as index
+        
+    R_aug : array or DataFrame (only if augment=True)
+        Augmented correlation matrix of shape (n_vars+1, n_vars+1) that includes
+        the composite as the last row/column.
+        
+    Algorithm Steps
+    ---------------
+    1. Extract R_yy: sub-matrix of correlations among composite items
+    2. Calculate r̄: mean of off-diagonal correlations in R_yy
+    3. Calculate denominator: σ_Y = sqrt(k + k(k-1)×r̄)
+    4. Calculate numerator: for each variable, sum its correlations with all composite items
+    5. Compute final correlation: r_comp = numerator / denominator
+    
+    Examples
+    --------
+    >>> # Simple example with 5 variables
+    >>> R = np.array([
+    ...     [1.0, 0.5, 0.6, 0.3, 0.4],
+    ...     [0.5, 1.0, 0.7, 0.2, 0.3],
+    ...     [0.6, 0.7, 1.0, 0.4, 0.5],
+    ...     [0.3, 0.2, 0.4, 1.0, 0.8],
+    ...     [0.4, 0.3, 0.5, 0.8, 1.0]
+    ... ])
+    >>> 
+    >>> # Create composite from first 3 variables (indices 0, 1, 2)
+    >>> r_comp = composite_correlations(R, composite_idx=[0, 1, 2])
+    >>> print(r_comp)
+    [0.95  0.95  0.95  0.48  0.60]  # Composite correlates highly with its items
+    
+    >>> # With variable names and augmented matrix
+    >>> var_names = ['Item1', 'Item2', 'Item3', 'Outcome1', 'Outcome2']
+    >>> r_comp, R_aug = composite_correlations(
+    ...     R, composite_idx=[0, 1, 2], var_names=var_names, augment=True
+    ... )
+    >>> print(r_comp)
+    Item1       0.95
+    Item2       0.95
+    Item3       0.95
+    Outcome1    0.48
+    Outcome2    0.60
+    Name: Composite, dtype: float64
+    
+    Notes
+    -----
+    - The composite items will have correlations close to 1.0 with the composite
+      (exact value depends on inter-item correlations)
+    - This assumes unit weighting (all items weighted equally)
+    - For reliability-weighted composites, use a different formula
+    - The denominator adjustment accounts for the fact that composite variance
+      includes both item variances and covariances
+      
+    References
+    ----------
+    - Nunnally, J. C., & Bernstein, I. H. (1994). Psychometric Theory (3rd ed.).
+      McGraw-Hill. Chapter 6: The Assessment of Reliability.
+    - Schmidt, F. L., & Hunter, J. E. (2015). Methods of Meta-Analysis (3rd ed.).
+      Sage Publications. Chapter 3: Correlational Artifacts.
+    """
+    # Convert input to numpy array (handles both arrays and DataFrames)
     R_mat = np.asarray(R, dtype=float)
-    n_all = R_mat.shape[0]
-    n = len(composite_idx)
-    # sub-matrix of the composite items
+    n_all = R_mat.shape[0]  # Total number of variables
+    n = len(composite_idx)   # Number of items in composite
+    
+    # STEP 1: Extract sub-matrix of correlations among composite items
+    # ----------------------------------------------------------------
+    # R_yy is the k×k correlation matrix for just the composite items
+    # Example: if composite_idx = [0, 2, 5] and R is 10×10,
+    #          R_yy will be the 3×3 matrix of correlations among variables 0, 2, 5
     R_yy = R_mat[np.ix_(composite_idx, composite_idx)]
-    # mean off-diagonal
+    
+    # STEP 2: Calculate average inter-item correlation (r̄)
+    # -----------------------------------------------------
+    # Get upper triangle indices (excluding diagonal)
+    # For a 3×3 matrix, this gives positions: (0,1), (0,2), (1,2)
     iu = np.triu_indices(n, k=1)
+    
+    # Extract off-diagonal correlations and compute mean
+    # This is the average correlation between items in the composite
+    # Example: if items correlate at [0.5, 0.6, 0.7], rbar = 0.6
     rbar = R_yy[iu].mean() if iu[0].size > 0 else 0.0
+    
+    # STEP 3: Calculate denominator (composite standard deviation)
+    # ------------------------------------------------------------
+    # Formula: σ_Y = sqrt(k + k(k-1)×r̄)
+    # 
+    # Derivation:
+    # For unit-weighted composite Y = X₁ + X₂ + ... + Xₖ (assuming standardized items):
+    #   Var(Y) = Var(X₁) + Var(X₂) + ... + Var(Xₖ) + 2×Σᵢ<ⱼ Cov(Xᵢ, Xⱼ)
+    #          = k + 2×Σᵢ<ⱼ r(Xᵢ, Xⱼ)
+    #          = k + k(k-1)×r̄
+    #   
+    #   where we used: Var(Xᵢ) = 1 (standardized)
+    #                  Cov(Xᵢ, Xⱼ) = r(Xᵢ, Xⱼ) (correlation = covariance for standardized vars)
+    #                  Number of pairs = k(k-1)/2, so 2×Σᵢ<ⱼ = k(k-1)×r̄
+    #
+    # Example: 3 items with r̄ = 0.6
+    #   denom = sqrt(3 + 3×2×0.6) = sqrt(3 + 3.6) = sqrt(6.6) ≈ 2.57
     denom = np.sqrt(n + n*(n-1)*rbar)
+    
+    # STEP 4: Calculate numerator (sum of correlations)
+    # -------------------------------------------------
+    # For each variable in the full matrix, sum its correlations with all composite items
+    # 
+    # R_mat[composite_idx, :] extracts rows corresponding to composite items
+    # .sum(axis=0) sums down columns, giving sum of correlations for each variable
+    #
+    # Example: If composite has items [A, B, C] and we want correlation with variable X:
+    #   numer[X] = r(A,X) + r(B,X) + r(C,X)
+    #
+    # This is vectorized - computes for all variables at once
     numer = R_mat[composite_idx, :].sum(axis=0)
+    
+    # STEP 5: Compute final composite correlation
+    # -------------------------------------------
+    # r(Composite, X) = Σᵢ r(Xᵢ, X) / σ_Y
+    #
+    # This divides the sum of item-X correlations by the composite's standard deviation
+    # The result is the correlation between the composite and each variable
+    #
+    # Interpretation:
+    # - Composite items will have r ≈ 0.9-1.0 (high correlation with their own composite)
+    # - Other variables will have r based on their average correlation with composite items
     r_comp = numer / denom
 
+    # Format output as pandas Series if variable names provided
     if var_names is not None:
         r_comp = pd.Series(r_comp, index=var_names, name="Composite")
 
+    # Return just correlations if augment=False
     if not augment:
         return r_comp
 
-    # build augmented matrix
+    # STEP 6: Build augmented correlation matrix (optional)
+    # -----------------------------------------------------
+    # Create a new correlation matrix that includes the composite as a new variable
+    # This is useful for further analyses that need the composite in the matrix
+    
     if var_names is not None:
+        # Create labeled DataFrame
         idx = list(var_names) + ["Composite"]
         R_aug = pd.DataFrame(np.zeros((n_all+1, n_all+1)), index=idx, columns=idx)
+        
+        # Copy original matrix to top-left block
         R_aug.iloc[:-1, :-1] = R_mat
-        R_aug.iloc[-1, :-1] = r_comp.values
-        R_aug.iloc[:-1, -1] = r_comp.values
+        
+        # Add composite correlations to last row and column
+        R_aug.iloc[-1, :-1] = r_comp.values  # Last row (composite vs all vars)
+        R_aug.iloc[:-1, -1] = r_comp.values  # Last column (all vars vs composite)
+        
+        # Diagonal element (composite vs itself) = 1.0
         R_aug.iloc[-1, -1] = 1.0
     else:
+        # Create unlabeled array
         R_aug = np.zeros((n_all+1, n_all+1))
         R_aug[:n_all, :n_all] = R_mat
         R_aug[n_all, :n_all] = r_comp
@@ -45,57 +235,220 @@ def composite_correlations(R, composite_idx, var_names=None, augment=False):
 
     return r_comp, R_aug
 
+
+# =============================================================================
+# STREAMLIT WEB APPLICATION
+# =============================================================================
+
 # Streamlit UI
 st.title("Composite-Correlation Calculator")
+
 st.markdown("""
-Upload a CSV containing your (possibly lower-triangular) correlation matrix.  
-The app will fill in missing cells by symmetry, set diagonals to 1,  
-then let you select which variables go into a unit-weighted composite.
+### What This Tool Does
+
+This calculator computes **unit-weighted composite correlations** from a correlation matrix.
+
+**Use Case:** You have a correlation matrix and want to:
+1. Combine multiple items into a composite score (e.g., sum of survey items)
+2. See how the composite correlates with other variables
+3. Add the composite to your correlation matrix for further analysis
+
+**How It Works:**
+- Upload a correlation matrix (CSV file)
+- Select which variables form the composite
+- Get correlations between the composite and all variables
+- Optionally get an augmented matrix with the composite included
+
+**Formula:** Uses the psychometric formula `r(Composite, X) = Σr(item, X) / sqrt(k + k(k-1)×r̄)`
+
+---
+
+### Instructions
+
+1. **Upload a CSV file** containing your correlation matrix
+   - First column should contain row labels (variable names)
+   - Can be lower-triangular (missing values will be filled by symmetry)
+   - Diagonal values will be set to 1.0 if missing
+
+2. **Select variables** to include in the composite (minimum 2)
+
+3. **Click "Compute"** to see results
 """)
 
 uploaded = st.file_uploader("Upload correlation matrix (CSV)", type=["csv"])
+
 if uploaded is not None:
-    # 1) read and label
+    # 1) Read and label the correlation matrix
     try:
         df = pd.read_csv(uploaded, index_col=0)
-    except Exception:
-        st.error("Failed to read CSV. Make sure the first column contains row labels.")
+    except Exception as e:
+        st.error(f"Failed to read CSV: {e}")
+        st.info("Make sure the first column contains row labels (variable names).")
         st.stop()
+    
+    # Validate square matrix
     if df.shape[0] != df.shape[1]:
-        st.error("Matrix must be square.")
+        st.error(f"Matrix must be square. Got {df.shape[0]} rows and {df.shape[1]} columns.")
         st.stop()
 
-    st.success(f"Loaded a {df.shape[0]}×{df.shape[1]} matrix.")
+    st.success(f"✅ Loaded a {df.shape[0]}×{df.shape[1]} correlation matrix.")
 
-    # 2) symmetrize and fill diagonal
+    # 2) Symmetrize and fill diagonal
+    # Many correlation matrices are stored as lower-triangular to save space
+    # This fills in the upper triangle by copying from the lower triangle
     mat = df.values.astype(float)
-    mat = np.where(np.isnan(mat), mat.T, mat)       # fill missing by transpose
-    np.fill_diagonal(mat, 1.0)                       # ensure diagonal = 1
+    mat = np.where(np.isnan(mat), mat.T, mat)  # Fill missing cells by transpose (symmetry)
+    np.fill_diagonal(mat, 1.0)                  # Ensure diagonal = 1.0 (self-correlation)
+    
     df_sym = pd.DataFrame(mat, index=df.index, columns=df.columns)
-    st.write("**Symmetrized & filled diagonal:**")
-    st.dataframe(df_sym)
+    
+    with st.expander("📊 View symmetrized correlation matrix"):
+        st.dataframe(df_sym.style.format("{:.3f}"))
 
-    # 3) select composite variables
+    # 3) Select composite variables
     all_vars = list(df_sym.columns)
+    
+    st.subheader("Select Composite Items")
+    st.markdown("Choose which variables to combine into a unit-weighted composite:")
+    
     composite_vars = st.multiselect(
-        "Select variables to form the composite",
+        "Variables in composite",
         options=all_vars,
-        default=all_vars[: min(3, len(all_vars))]
+        default=all_vars[: min(3, len(all_vars))],  # Default to first 3 variables
+        help="Select at least 2 variables to form a composite"
     )
+    
     if len(composite_vars) < 2:
-        st.warning("Pick at least 2 variables for a composite.")
+        st.warning("⚠️ Please select at least 2 variables to form a composite.")
     else:
-        if st.button("Compute composite correlations"):
+        st.info(f"Selected {len(composite_vars)} items for composite: {', '.join(composite_vars)}")
+        
+        if st.button("🧮 Compute Composite Correlations", type="primary"):
+            # Get indices of selected variables
             idx = [all_vars.index(v) for v in composite_vars]
+            
+            # Compute composite correlations with augmented matrix
             r_comp, R_aug = composite_correlations(
                 df_sym.values,
                 composite_idx=idx,
                 var_names=all_vars,
                 augment=True
             )
-            st.subheader("Composite vs. Each Variable")
-            st.dataframe(r_comp.to_frame())
-            st.subheader("Augmented Correlation Matrix")
-            st.dataframe(R_aug)
+            
+            # Display results
+            st.success("✅ Computation complete!")
+            
+            # Show composite correlations
+            st.subheader("📈 Composite Correlations")
+            st.markdown("""
+            These are the correlations between your composite (sum of selected items) 
+            and each variable in the matrix.
+            """)
+            
+            # Create a styled dataframe
+            result_df = r_comp.to_frame()
+            result_df.columns = ['Correlation with Composite']
+            
+            # Highlight composite items
+            def highlight_composite(row):
+                if row.name in composite_vars:
+                    return ['background-color: #e3f2fd'] * len(row)
+                return [''] * len(row)
+            
+            st.dataframe(
+                result_df.style
+                .format("{:.4f}")
+                .apply(highlight_composite, axis=1)
+                .bar(subset=['Correlation with Composite'], color='#1f77b4', vmin=-1, vmax=1)
+            )
+            
+            st.caption("💡 Composite items (highlighted) typically have high correlations (0.8-1.0) with the composite.")
+            
+            # Show augmented matrix
+            st.subheader("📊 Augmented Correlation Matrix")
+            st.markdown("""
+            This matrix includes your composite as a new variable (last row/column).
+            You can use this for further analyses.
+            """)
+            
+            with st.expander("View full augmented matrix"):
+                st.dataframe(R_aug.style.format("{:.3f}"))
+            
+            # Download options
+            st.subheader("💾 Download Results")
+            
+            col1, col2 = st.columns(2)
+            
+            with col1:
+                # Download composite correlations
+                csv1 = result_df.to_csv()
+                st.download_button(
+                    label="Download Composite Correlations (CSV)",
+                    data=csv1,
+                    file_name="composite_correlations.csv",
+                    mime="text/csv"
+                )
+            
+            with col2:
+                # Download augmented matrix
+                csv2 = R_aug.to_csv()
+                st.download_button(
+                    label="Download Augmented Matrix (CSV)",
+                    data=csv2,
+                    file_name="augmented_correlation_matrix.csv",
+                    mime="text/csv"
+                )
+            
+            # Show interpretation guide
+            with st.expander("📖 How to Interpret Results"):
+                st.markdown("""
+                **Composite Correlations:**
+                - **High (0.7-1.0):** Strong relationship with composite
+                - **Moderate (0.3-0.7):** Moderate relationship
+                - **Low (0.0-0.3):** Weak relationship
+                - **Negative:** Inverse relationship
+                
+                **Composite Items:**
+                - Should correlate highly (0.8-1.0) with the composite
+                - Lower values suggest the item doesn't fit well
+                - Consider removing items with r < 0.7
+                
+                **Other Variables:**
+                - Correlation shows how well they relate to the composite
+                - Use for criterion validity, predictive validity, etc.
+                
+                **Formula Used:**
+                ```
+                r(Composite, X) = Σr(item, X) / sqrt(k + k(k-1)×r̄)
+                ```
+                where k = number of items, r̄ = average inter-item correlation
+                """)
+
 else:
-    st.info("🤖 Upload a CSV file to get started.")
+    st.info("👆 Upload a CSV file containing your correlation matrix to get started.")
+    
+    # Show example
+    with st.expander("📝 Example CSV Format"):
+        st.markdown("""
+        Your CSV should look like this:
+        
+        ```
+        ,Item1,Item2,Item3,Outcome1,Outcome2
+        Item1,1.0,0.5,0.6,0.3,0.4
+        Item2,0.5,1.0,0.7,0.2,0.3
+        Item3,0.6,0.7,1.0,0.4,0.5
+        Outcome1,0.3,0.2,0.4,1.0,0.8
+        Outcome2,0.4,0.3,0.5,0.8,1.0
+        ```
+        
+        Or lower-triangular (missing values will be filled):
+        
+        ```
+        ,Item1,Item2,Item3,Outcome1,Outcome2
+        Item1,1.0,,,,
+        Item2,0.5,1.0,,,
+        Item3,0.6,0.7,1.0,,
+        Outcome1,0.3,0.2,0.4,1.0,
+        Outcome2,0.4,0.3,0.5,0.8,1.0
+        ```
+        """)