Update app.py

app.py

@@ -391,120 +391,146 @@ if not df_final.empty and source_choice == "(All)":
 # ------------------- Output -------------------
 # fig = plot_radar(df_final, grouped, chart_title)
 # st.plotly_chart(fig, use_container_width=True)
-# ============== Build per-stage vectors for comparisons ==============
+# ============== Build per-stage vectors for comparisons (LIKERT-AWARE) ==============
+
 # Columns to use based on mode
 COLS = list(SKILL_GROUPS.keys()) if grouped else SKILLS
 
-# Helper to extract the mean vector for (student, source) from df_resp/df_final
-def _mean_vector_for(student: str | None, source: str | None, use_merged: bool) -> dict:
-    """
-    use_merged=True -> read from df_final (after Likert merge)
-    use_merged=False -> read from df_resp (responses-only)
-    """
-    df_base = df_final if use_merged else df_resp
-    if df_base.empty:
-        return {k: None for k in COLS}
-
-    if student and source:
-        label = f"{student} — {source}"
-        sub = df_base[df_base["label"] == label]
-    elif student and source is None:
-        # combined sources row (when overlay OFF)
-        sub = df_base[df_base["label"] == student]
-    else:
-        # cohort average across all rows in df_base
-        sub = df_base
+# Map each skill to its group(s) once (used to project group Likert down to skills)
+SKILL_TO_GROUPS = {s: [g for g, members in SKILL_GROUPS.items() if s in members] for s in SKILLS}
 
-    if sub.empty:
-        return {k: None for k in COLS}
-    means = sub[COLS].mean(numeric_only=True)
-    return {k: (None if pd.isna(means.get(k)) else float(means.get(k))) for k in COLS}
+def _project_likert_to_cols(likert_grouped: dict | None, cols: list[str], grouped_flag: bool) -> dict:
+    """Return a vector aligned to COLS from group-level Likert. If ungrouped, project to skills."""
+    if not likert_grouped:
+        return {k: None for k in cols}
+    if grouped_flag:
+        return {k: (likert_grouped.get(k) if k in likert_grouped else None) for k in cols}
+    # ungrouped → average the groups a skill belongs to
+    out = {}
+    for s in cols:
+        gs = SKILL_TO_GROUPS.get(s, [])
+        vals = [likert_grouped.get(g) for g in gs if likert_grouped.get(g) is not None]
+        out[s] = float(np.mean(vals)) if vals else None
+    return out
 
-# Build mapping skill->groups (you already used this in the Likert merge)
-SKILL_TO_GROUPS = {s: [g for g, members in SKILL_GROUPS.items() if s in members] for s in SKILLS}
+def _merge_resp_and_likert(resp_vec: dict, likert_vec: dict) -> dict:
+    """Average where both exist; else take whichever exists."""
+    out = {}
+    for k in resp_vec.keys():
+        rv = resp_vec.get(k, None)
+        lv = likert_vec.get(k, None)
+        if rv is not None and lv is not None:
+            out[k] = (rv + lv) / 2.0
+        elif rv is not None:
+            out[k] = rv
+        else:
+            out[k] = lv
+    return out
+
+def _mean_vectors(vecs: list[dict]) -> dict:
+    """Element-wise mean ignoring None; returns None if all Nones for a key."""
+    if not vecs:
+        return {}
+    keys = list(vecs[0].keys())
+    out = {}
+    for k in keys:
+        vals = [v.get(k) for v in vecs if v.get(k) is not None]
+        out[k] = (float(np.mean(vals)) if vals else None)
+    return out
+
+def _resp_mean_for_sources(df_src: pd.DataFrame, student: str | None, sources: list[str], cols: list[str]) -> dict:
+    """Mean of response scores across docs for (student,sources). If student None → cohort."""
+    if df_src.empty:
+        return {k: None for k in cols}
+    sub = df_src.copy()
+    if student:
+        sub = sub[sub["student"] == student]
+    sub = sub[sub["source"].isin(sources)]
+    if sub.empty:
+        return {k: None for k in cols}
+    m = sub[cols].mean(numeric_only=True)
+    return {k: (None if pd.isna(m.get(k)) else float(m.get(k))) for k in cols}
 
-def _likert_grouped_for(student: str, stage: str) -> dict | None:
+def _likert_grouped_for_student_stage(student: str, stage: str) -> dict | None:
+    """Get normalized (0–1) group-level Likert for onboarding/closing only."""
     if stage not in ("onboarding", "closing"):
         return None
     lg = mongo_get_likert_grouped(mongo_uri, db_name, summaries_coll, student, stage)
     return lg if lg else None
 
-def _stage_vector(student: str | None, stage: str) -> dict:
-    # Which sources make up this stage?
-    if stage == "onboarding":
-        srcs = ["onboarding_responses"]
-    elif stage == "closing":
-        srcs = ["closing_responses"]
-    elif stage == "combined_weeks":
-        srcs = ["week_2_responses", "week_3_responses", "closing_responses"]
-    else:
-        srcs = []
-
-    # Response-only mean across those sources
-    if not df_resp.empty:
-        if student and source_choice == "(All)":
-            # we may have aggregated to one row per student; compute from df_raw instead
-            # build per-source labels then average
-            rows = []
-            for s in srcs:
-                lbl = f"{student} — {s}"
-                sub = df_resp[df_resp["label"] == lbl]
-                if not sub.empty:
-                    rows.append(sub[COLS].mean(numeric_only=True))
-            if rows:
-                m = pd.concat(rows, axis=1).mean(axis=1)
-                resp_vec = {k: (None if pd.isna(m.get(k)) else float(m.get(k))) for k in COLS}
-            else:
-                resp_vec = {k: None for k in COLS}
-        elif student and source_choice != "(All)":
-            # if the UI is filtered to a specific source, ignore that and recompute from df_resp
-            rows = []
-            for s in srcs:
-                lbl = f"{student} — {s}"
-                sub = df_resp[df_resp["label"] == lbl]
-                if not sub.empty:
-                    rows.append(sub[COLS].mean(numeric_only=True))
-            if rows:
-                m = pd.concat(rows, axis=1).mean(axis=1)
-                resp_vec = {k: (None if pd.isna(m.get(k)) else float(m.get(k))) for k in COLS}
-            else:
-                resp_vec = {k: None for k in COLS}
-        else:
-            # cohort: average across all matching sources
-            sub = df_resp[df_resp["label"].str.contains(" — ", na=False)]
-            sub = sub[sub["label"].str.split(" — ").str[1].isin(srcs)]
-            if not sub.empty:
-                m = sub[COLS].mean(numeric_only=True)
-                resp_vec = {k: (None if pd.isna(m.get(k)) else float(m.get(k))) for k in COLS}
-            else:
-                resp_vec = {k: None for k in COLS}
-    else:
-        resp_vec = {k: None for k in COLS}
+def _student_stage_vectors(df_src: pd.DataFrame, stu: str, cols: list[str], grouped_flag: bool) -> dict:
+    """Per-student vectors with Likert merged for onboarding/closing; combined includes closing(merged)."""
+    # Onboarding = RESP(onboarding) ⊕ Likert(onboarding)
+    onb_resp = _resp_mean_for_sources(df_src, stu, ["onboarding_responses"], cols)
+    onb_lik  = _project_likert_to_cols(_likert_grouped_for_student_stage(stu, "onboarding"), cols, grouped_flag)
+    onb = _merge_resp_and_likert(onb_resp, onb_lik)
 
-    # Merge in Likert for onboarding/closing (projected to skills if ungrouped)
-    if student:
-        likert_g = _likert_grouped_for(student, "onboarding" if "onboarding_responses" in srcs else ("closing" if "closing_responses" in srcs and len(srcs)==1 else None))
-    else:
-        likert_g = None  # no cohort Likert
+    # Closing = RESP(closing) ⊕ Likert(closing)
+    cls_resp = _resp_mean_for_sources(df_src, stu, ["closing_responses"], cols)
+    cls_lik  = _project_likert_to_cols(_likert_grouped_for_student_stage(stu, "closing"), cols, grouped_flag)
+    cls = _merge_resp_and_likert(cls_resp, cls_lik)
+
+    # Combined = mean( RESP(week2), RESP(week3), CLOSING(merged) )
+    w2 = _resp_mean_for_sources(df_src, stu, ["week_2_responses"], cols)
+    w3 = _resp_mean_for_sources(df_src, stu, ["week_3_responses"], cols)
+    combo = _mean_vectors([w2, w3, cls])  # <- note: closing already merged with Likert
+
+    return {"onboarding": onb, "closing": cls, "combined": combo}
+
+def _stage_vectors_for_current_selection(df_src: pd.DataFrame, student_choice: str | None, cols: list[str], grouped_flag: bool) -> dict:
+    """
+    If a student is selected → return their vectors.
+    If cohort (“(All)”) → average per-student vectors (Likert included where available).
+    """
+    if student_choice and student_choice != "(All)":
+        return _student_stage_vectors(df_src, student_choice, cols, grouped_flag)
+
+    # Cohort: compute for each student then average
+    if df_src.empty:
+        empty_vec = {k: None for k in cols}
+        return {"onboarding": empty_vec, "closing": empty_vec, "combined": empty_vec}
+
+    students = sorted(set(str(x) for x in df_src["student"].dropna().unique()))
+    per_student = [_student_stage_vectors(df_src, s, cols, grouped_flag) for s in students]
+    return {
+        "onboarding": _mean_vectors([p["onboarding"] for p in per_student]),
+        "closing":    _mean_vectors([p["closing"]    for p in per_student]),
+        "combined":   _mean_vectors([p["combined"]   for p in per_student]),
+    }
 
-    merged = _merge_resp_and_likert_vector(resp_vec, likert_g, grouped, SKILL_TO_GROUPS, SKILL_GROUPS)
-    return merged
+def _percent_change(new: float | None, old: float | None) -> float | None:
+    if new is None or old is None:
+        return None
+    if old == 0:
+        return None  # or return 100.0 if you prefer
+    return (new - old) / old * 100.0
 
-# Build the vectors we need
-if student_choice != "(All)":
-    vec_onb = _stage_vector(student_choice, "onboarding")
-    vec_cls = _stage_vector(student_choice, "closing")
-    vec_combo = _stage_vector(student_choice, "combined_weeks")
+# Use df_raw (one row per doc) so overlay/aggregation doesn’t hide sources
+# Ensure df_raw has the per-skill or per-group columns we need:
+if grouped and not df_raw.empty:
+    # build grouped view just for comparisons
+    df_grouped_for_comp = df_raw.copy()
+    # aggregate per-doc row to grouped columns
+    df_grouped_for_comp = (
+        df_grouped_for_comp
+        .assign(**{
+            g: df_grouped_for_comp.apply(lambda r: safe_mean([r.get(s, np.nan) for s in SKILL_GROUPS[g]]), axis=1)
+            for g in SKILL_GROUPS.keys()
+        })
+    )
+    df_src_for_comp = df_grouped_for_comp[["student", "source"] + list(SKILL_GROUPS.keys())]
 else:
-    # Cohort-wide comparison
-    vec_onb = _stage_vector(None, "onboarding")
-    vec_cls = _stage_vector(None, "closing")
-    vec_combo = _stage_vector(None, "combined_weeks")
+    df_src_for_comp = df_raw  # already per-skill
 
-# Compute % deltas
-pct_onb_to_cls = {k: _percent_change(vec_cls.get(k), vec_onb.get(k)) for k in COLS}
+stage_vecs = _stage_vectors_for_current_selection(df_src_for_comp, student_choice, COLS, grouped)
+vec_onb   = stage_vecs["onboarding"]
+vec_cls   = stage_vecs["closing"]
+vec_combo = stage_vecs["combined"]
+
+pct_onb_to_cls   = {k: _percent_change(vec_cls.get(k),   vec_onb.get(k)) for k in COLS}
 pct_onb_to_combo = {k: _percent_change(vec_combo.get(k), vec_onb.get(k)) for k in COLS}
 
+# ------------------- Plot + table above stays the same -------------------
 df_plot = df_final.copy()
 avg_label = None
 
@@ -524,6 +550,61 @@ st.plotly_chart(fig, use_container_width=True)
 
 st.caption(f"{len(df_final)} line(s) aggregated." if not df_final.empty else "No data.")
 
+# ------------------- Comparisons Tab (LIKERT-AWARE) -------------------
+tab_compare, = st.tabs(["📊 Comparisons"])
+
+with tab_compare:
+    st.subheader("Onboarding → Closing — % Change (Likert-aware)")
+    df1 = pd.DataFrame({
+        "Dimension": COLS,
+        "Onboarding (merged)": [vec_onb.get(k) for k in COLS],
+        "Closing (merged)":    [vec_cls.get(k) for k in COLS],
+        "% Change":            [pct_onb_to_cls.get(k) for k in COLS],
+    })
+    st.dataframe(
+        df1.style.format({
+            "Onboarding (merged)": "{:.2f}",
+            "Closing (merged)":    "{:.2f}",
+            "% Change":            "{:+.1f}%"
+        }),
+        use_container_width=True
+    )
+
+    st.subheader("Onboarding → (Week2 + Week3 + ClosingMerged) — % Change")
+    df2 = pd.DataFrame({
+        "Dimension": COLS,
+        "Onboarding (merged)":                   [vec_onb.get(k) for k in COLS],
+        "Weeks 2+3 + Closing (closing merged)":  [vec_combo.get(k) for k in COLS],
+        "% Change":                              [pct_onb_to_combo.get(k) for k in COLS],
+    })
+    st.dataframe(
+        df2.style.format({
+            "Onboarding (merged)":                  "{:.2f}",
+            "Weeks 2+3 + Closing (closing merged)": "{:.2f}",
+            "% Change":                             "{:+.1f}%"
+        }),
+        use_container_width=True
+    )
+
+    # Optional bar chart: %Δ Onboarding → Closing
+    try:
+        fig_delta = go.Figure()
+        fig_delta.add_bar(
+            x=COLS,
+            y=[(pct_onb_to_cls.get(k) if pct_onb_to_cls.get(k) is not None else 0.0) for k in COLS],
+            name="%Δ Onb→Closing"
+        )
+        fig_delta.update_layout(
+            title="% Change: Onboarding → Closing (Likert-aware)",
+            xaxis_title="Dimension",
+            yaxis_title="% change",
+            margin=dict(l=20, r=20, t=50, b=20)
+        )
+        st.plotly_chart(fig_delta, use_container_width=True)
+    except Exception:
+        pass
+
+
 
 # ================== Dynamic Stage Summaries (only if student answered that week) ==================
 import re