Update app.py

app.py

@@ -47,12 +47,15 @@ from huggingface_hub import login
 
 from safety import safety_filter, refusal_reply
 from retriever import init_retriever, retrieve_context
-from decision_math import compute_operitional_numbers as compute_operational_numbers  # in case of rename
+
+# ---------- Snapshot & retrieval helpers import ----------
+# Use the real function if present; otherwise fall back to a harmless no-op.
 try:
-    # prefer the original name if present
-    from decision_math import compute_operational_numbers as compute_operational_numbers
+    from decision_math import compute_operational_numbers
 except Exception:
-    pass
+    def compute_operational_numbers(snapshot: dict) -> dict:
+        return {}
+
 from prompt_templates import build_system_preamble
 from upload_ingest import extract_text_from_files
 from session_rag import SessionRAG
@@ -235,237 +238,6 @@ def _mdsi_block():
         "outcomes_summary": outcomes
     }, indent=2)
 
-# ---------- Scenario-agnostic dynamic analytics (multi-file) ----------
-import pandas as _pd
-from collections import Counter
-import itertools as _it
-import numpy as _np
-
-_NUM_SAMPLE_ROWS = 50000  # cap per file for speed
-
-def _read_table(path: str) -> _pd.DataFrame:
-    try:
-        if path.lower().endswith((".xlsx", ".xls")):
-            return _pd.read_excel(path)
-        return _pd.read_csv(path, low_memory=False, nrows=_NUM_SAMPLE_ROWS)
-    except Exception:
-        return _pd.DataFrame()
-
-def _profile_schema(df: _pd.DataFrame) -> Dict[str, Any]:
-    if df.empty:
-        return {"numeric": [], "categorical": [], "datetime": [], "textlike": []}
-    numeric, categorical, datetime, textlike = [], [], [], []
-    for c in df.columns:
-        s = df[c]
-        if _pd.api.types.is_numeric_dtype(s):
-            numeric.append(c)
-        elif _pd.api.types.is_datetime64_any_dtype(s):
-            datetime.append(c)
-        else:
-            uniq = s.astype(str).nunique(dropna=True)
-            if uniq <= max(50, int(0.03 * max(1, len(s)))):
-                categorical.append(c)
-            else:
-                textlike.append(c)
-    return {"numeric": numeric, "categorical": categorical, "datetime": datetime, "textlike": textlike}
-
-def _safe_num(s: _pd.Series) -> _pd.Series:
-    if not _pd.api.types.is_numeric_dtype(s):
-        return _pd.to_numeric(s, errors="coerce")
-    return s
-
-def _top_numeric_by_variance(df: _pd.DataFrame, numeric_cols: List[str], k=5) -> List[str]:
-    scores = []
-    for c in numeric_cols:
-        x = _safe_num(df[c])
-        try:
-            scores.append((c, _np.nanvar(x.values)))
-        except Exception:
-            scores.append((c, _np.nan))
-    scores.sort(key=lambda t: (t[1] if _np.isfinite(t[1]) else -1), reverse=True)
-    return [c for c, _ in scores[:k]]
-
-def _top_categories(df: _pd.DataFrame, cat_cols: List[str], k=3) -> Dict[str, List[Tuple[str,int]]]:
-    out = {}
-    for c in cat_cols[:6]:
-        vc = Counter(df[c].astype(str).fillna("<NA>")).most_common(k)
-        out[c] = vc
-    return out
-
-def _infer_candidate_keys(dfs_named: List[Tuple[str, _pd.DataFrame]]) -> List[str]:
-    all_cols = []
-    for name, df in dfs_named:
-        all_cols.extend(list(map(str, df.columns)))
-    counts = Counter([c.strip() for c in all_cols])
-    bad = set(["value","values","count","total","sum","mean","median","date","timestamp","index"])
-    return [c for c, n in counts.items() if n >= 2 and c.lower() not in bad]
-
-def _try_joins(dfs_named: List[Tuple[str, _pd.DataFrame]], keys: List[str], max_pairs=3) -> List[str]:
-    previews = []
-    pairs = list(_it.combinations(range(len(dfs_named)), 2))
-    shown = 0
-    for i, j in pairs:
-        if shown >= max_pairs:
-            break
-        name_i, dfi = dfs_named[i]
-        name_j, dfj = dfs_named[j]
-        for k in keys:
-            if k in dfi.columns and k in dfj.columns:
-                try:
-                    merged = dfi[[k]].dropna().merge(dfj[[k]].dropna(), on=k, how="inner")
-                    previews.append(f"- Join {name_i} ↔ {name_j} on `{k}` → matches: {len(merged):,}")
-                    shown += 1
-                    if shown >= max_pairs:
-                        break
-                except Exception:
-                    continue
-    return previews
-
-def _scenario_tokens(text: str) -> List[str]:
-    t = (text or "").lower()
-    t = re.sub(r"[^a-z0-9_ -]+", " ", t)
-    toks = [w for w in t.split() if len(w) >= 3]
-    out, seen = [], set()
-    for w in toks:
-        if w not in seen:
-            seen.add(w); out.append(w)
-    return out
-
-def _extract_intents(text: str) -> Dict[str, Any]:
-    toks = _scenario_tokens(text)
-    intents = {
-        "rank": any(x in toks for x in ["rank","top","longest","highest","lowest","shortest","worst","best"]),
-        "agg_words": [w for w in toks if w in set(["mean","average","avg","median","p50","p90","sum","total"])],
-        "n_top": 5
-    }
-    return intents
-
-def _pick_dims_from_tokens(df: _pd.DataFrame, cat_cols: List[str], toks: List[str]) -> List[str]:
-    scored = []
-    for c in cat_cols:
-        score = sum(1 for t in toks if t in c.lower())
-        scored.append((score, c))
-    scored.sort(key=lambda t: (t[0], -len(t[1])), reverse=True)
-    picked = [c for s, c in scored if s > 0][:3]
-    if not picked:
-        picked = cat_cols[:3]
-    return picked
-
-def _pick_metrics_from_tokens(df: _pd.DataFrame, num_cols: List[str], toks: List[str]) -> List[str]:
-    scored = []
-    for c in num_cols:
-        score = sum(1 for t in toks if t in c.lower())
-        scored.append((score, c))
-    scored.sort(key=lambda t: (t[0], -len(t[1])), reverse=True)
-    picked = [c for s, c in scored if s > 0][:3]
-    if not picked:
-        picked = _top_numeric_by_variance(df, num_cols, k=3)
-    return picked
-
-def _mk_table(md_title: str, df: _pd.DataFrame, limit=10) -> str:
-    if df.empty: return ""
-    return f"{md_title}\n" + df.head(limit).to_markdown(index=False)
-
-def compute_dynamic_analytics_block(arts: List[Dict[str, Any]], scenario_text: str) -> str:
-    dfs_named: List[Tuple[str, _pd.DataFrame]] = []
-    for a in arts or []:
-        p = a.get("path"); n = a.get("name") or "table"
-        if not p: continue
-        if not str(p).lower().endswith((".csv",".xlsx",".xls")): continue
-        d = _read_table(p)
-        if d.empty: continue
-        d = d.copy()
-        d.columns = [str(c).strip().replace("\n"," ").replace("\r"," ") for c in d.columns]
-        dfs_named.append((n, d))
-
-    if not dfs_named:
-        return ""
-
-    overview_rows = []
-    for n, d in dfs_named:
-        overview_rows.append({"File": n, "Rows": len(d), "Columns": d.shape[1]})
-    overview_md = _pd.DataFrame(overview_rows).to_markdown(index=False)
-
-    per_table_blocks = []
-    toks = _scenario_tokens(scenario_text)
-    intents = _extract_intents(scenario_text)
-
-    for n, d in dfs_named:
-        prof = _profile_schema(d)
-        num_cols = prof["numeric"]
-        cat_cols = prof["categorical"]
-
-        top_num = _top_numeric_by_variance(d, num_cols, k=5) if num_cols else []
-        num_sum = _pd.DataFrame()
-        if top_num:
-            stat_rows = []
-            for c in top_num:
-                x = _safe_num(d[c])
-                try_mean = float(_np.nanmean(x)) if x.size else _np.nan
-                try_median = float(_np.nanmedian(x)) if x.size else _np.nan
-                try_p90 = float(_np.nanpercentile(x.dropna(), 90)) if x.dropna().size else _np.nan
-                stat_rows.append({
-                    "Metric": c,
-                    "count": int(x.count()),
-                    "mean": try_mean,
-                    "median": try_median,
-                    "p90": try_p90
-                })
-            num_sum = _pd.DataFrame(stat_rows)
-
-        cat_info = _top_categories(d, cat_cols, k=5) if cat_cols else {}
-        cat_md = []
-        for c, vc in cat_info.items():
-            parts = ", ".join([f"{val} ({cnt})" for val, cnt in vc])
-            cat_md.append(f"- {c}: {parts}")
-
-        rank_tables = []
-        if intents.get("rank") and num_cols and cat_cols:
-            dims = _pick_dims_from_tokens(d, cat_cols, toks)
-            mets = _pick_metrics_from_tokens(d, num_cols, toks)
-            for gcol in dims[:2]:
-                for mcol in mets[:2]:
-                    try:
-                        g = (
-                            d.groupby(gcol, as_index=False)[mcol]
-                             .mean(numeric_only=True)
-                             .rename(columns={mcol: f"avg({mcol})"})
-                             .sort_values(f"avg({mcol})", ascending=False)
-                             .head(intents["n_top"])
-                        )
-                        rank_tables.append(_mk_table(f"Top {intents['n_top']} by avg({mcol}) — grouped by {gcol}:", g))
-                    except Exception:
-                        continue
-
-        block_parts = [f"### {n}"]
-        if not num_sum.empty:
-            block_parts.append(_mk_table("Numeric summary (top-variance metrics):", num_sum))
-        if cat_md:
-            block_parts.append("Top categories:\n" + "\n".join(cat_md))
-        for rt in rank_tables:
-            if rt: block_parts.append(rt)
-
-        per_table_blocks.append("\n\n".join([p for p in block_parts if p]))
-
-    keys = _infer_candidate_keys(dfs_named)
-    join_md = ""
-    if keys:
-        joins = _try_joins(dfs_named, keys, max_pairs=3)
-        if joins:
-            join_md = "Join previews:\n" + "\n".join(joins)
-
-    parts = [
-        "Computed Analytics Block (auto-generated, scenario-agnostic):",
-        "",
-        "Dataset overview:",
-        overview_md,
-        "",
-        "\n\n".join(per_table_blocks)
-    ]
-    if join_md:
-        parts.extend(["", join_md])
-    return "\n".join(parts)
-
 # ---------- Dynamic Phase 1 question generator ----------
 def _extract_present_domains(artifacts: List[Dict[str, Any]]) -> Dict[str, bool]:
     flags = dict(population=False, cost=False, clinical=False, capacity=False)
@@ -629,7 +401,6 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
             })
             return history + [(user_msg, phase1)], True
 
-        # ---------- Phase 2 ----------
         session_snips = "\n---\n".join(_session_rag.retrieve(
             "diabetes screening Indigenous Métis mobile program cost throughput outcomes logistics",
             k=6
@@ -656,20 +427,12 @@ def clarityops_reply(user_msg, history, tz, uploaded_files_paths, awaiting_answe
         else:
             artifact_block = "Uploaded Data Files (summarized):\n- <none>"
 
-        # NEW: scenario-agnostic, multi-file analytics block
-        analytics_block = compute_dynamic_analytics_block(arts, safe_in)
-
         scenario_block = safe_in if len((safe_in or "")) > 0 else ""
         system_preamble = build_system_preamble(
             snapshot=snapshot,
             policy_context=policy_context,
             computed_numbers=computed,
-            scenario_text=(
-                scenario_block
-                + f"\n\n{artifact_block}"
-                + (f"\n\n{analytics_block}" if analytics_block else "")
-                + (f"\n\nExecutive Pre-Computed Blocks:\n{mdsi_extra}" if mdsi_extra else "")
-            ),
+            scenario_text=scenario_block + f"\n\n{artifact_block}" + (f"\n\nExecutive Pre-Computed Blocks:\n{mdsi_extra}" if mdsi_extra else ""),
             session_snips=session_snips
         )
 
@@ -857,3 +620,4 @@ if __name__ == "__main__":
 
 
 
+