Add complaint sites filtering to overview tables with dedicated tab, separate complaint-specific tables for multi-RAT summary and top anomalies, and include complaint data in Excel export with two additional sheets

panel_app/kpi_health_check_panel.py

@@ -417,7 +417,6 @@ except Exception:  # noqa: BLE001
         rules_table.configuration = cfg
     except Exception:  # noqa: BLE001
         pass
-
 site_summary_table = pn.widgets.Tabulator(
     height=260, sizing_mode="stretch_width", layout="fit_data_table"
 )
@@ -426,10 +425,18 @@ multirat_summary_table = pn.widgets.Tabulator(
     height=260, sizing_mode="stretch_width", layout="fit_data_table"
 )
 
+complaint_multirat_summary_table = pn.widgets.Tabulator(
+    height=260, sizing_mode="stretch_width", layout="fit_data_table"
+)
+
 top_anomalies_table = pn.widgets.Tabulator(
     height=260, sizing_mode="stretch_width", layout="fit_data_table"
 )
 
+complaint_top_anomalies_table = pn.widgets.Tabulator(
+    height=260, sizing_mode="stretch_width", layout="fit_data_table"
+)
+
 site_select = pn.widgets.AutocompleteInput(
     name="Select a site (Type to search)",
     options={},
@@ -446,7 +453,6 @@ kpi_compare_norm = pn.widgets.Select(
     name="Normalization", options=["None", "Min-Max", "Z-score"], value="None"
 )
 
-# NEW WIDGETS
 kpi_group_select = pn.widgets.Select(
     name="KPI Group", options=["All (selected KPIs)"], value="All (selected KPIs)"
 )
@@ -535,6 +541,8 @@ except Exception:  # noqa: BLE001
 _set_tabulator_pagination(site_summary_table, page_size=50)
 _set_tabulator_pagination(multirat_summary_table, page_size=50)
 _set_tabulator_pagination(top_anomalies_table, page_size=50)
+_set_tabulator_pagination(complaint_multirat_summary_table, page_size=50)
+_set_tabulator_pagination(complaint_top_anomalies_table, page_size=50)
 _set_tabulator_pagination(site_kpi_table, page_size=50)
 trend_plot_pane = pn.pane.Plotly(sizing_mode="stretch_both", config=PLOTLY_CONFIG)
 heatmap_plot_pane = pn.pane.Plotly(sizing_mode="stretch_both", config=PLOTLY_CONFIG)
@@ -780,7 +788,6 @@ def _update_kpi_options() -> None:
     ]
     kpis = sorted([str(c) for c in kpis])
 
-    # Apply Grouping if needed
     groups = get_kpis_by_group(kpis)
     group_options = ["All (selected KPIs)"] + sorted(
         [g for g in groups.keys() if g != "Other"]
@@ -794,7 +801,6 @@ def _update_kpi_options() -> None:
         if kpi_group_select.value not in group_options:
             kpi_group_select.value = group_options[0]
 
-        # Filter KPIs based on group
         filtered_kpis = filter_kpis(
             kpis, kpi_group_select.value, mode=kpi_group_mode.value
         )
@@ -852,20 +858,14 @@ def _update_site_view(event=None) -> None:
     except Exception:  # noqa: BLE001
         available_sites = set()
 
-    # Robustly resolve code_int from site_select.value
-    # AutocompleteInput might return the Label (str) or Value (int) depending on usage
     code_int = None
     if code is not None:
-        # 1. Try if code is already a known value (int)
         if hasattr(site_select, "options") and isinstance(site_select.options, dict):
-            # Check if it matches a Key (Label)
             if code in site_select.options:
                 code_int = site_select.options[code]
-            # Check if it is a Value in the dict
             elif code in site_select.options.values():
                 code_int = code
 
-        # 2. If not checking opts or not found, try direct cast
         if code_int is None:
             try:
                 code_int = int(code)
@@ -932,36 +932,38 @@ def _update_site_view(event=None) -> None:
         trend_plot_pane.object, heatmap_plot_pane.object, hist_plot_pane.object = cached
         return
 
-    # Determine KPIs to plot based on group mode
     kpis_to_plot = []
 
-    # 1. Start with explicitly selected 'Compare KPIs'
     selected = [str(x) for x in (kpi_compare_select.value or []) if str(x)]
 
-    # 2. Add the primary selected KPI if not present
     if kpi and str(kpi) not in selected:
         selected = [str(kpi)] + selected
 
-    # 3. Handle Group Mode "Add top 12 KPIs"
-    # If mode is "Add top...", we fetch from group and append
     if "Add top" in str(kpi_group_mode.value):
         from_group = filter_kpis(
             d.columns.tolist(), kpi_group_select.value, mode="Top-N", top_n=12
         )
-        # Merge unique
         for gk in from_group:
             if gk not in selected:
                 selected.append(gk)
 
-    # Safeguard: Limit to 15 max to prevent browser crash
     kpis_to_plot = selected[:15]
 
-    # Build Plot using new module
-    # We need the rules for this RAT/KPIs to show SLA
-    relevant_rules = pd.DataFrame()
-    if isinstance(rules_table.value, pd.DataFrame) and not rules_table.value.empty:
-        r = rules_table.value
-        relevant_rules = r[r["RAT"] == rat]
+    rules_df = (
+        rules_table.value
+        if isinstance(rules_table.value, pd.DataFrame)
+        else pd.DataFrame()
+    )
+    relevant_rules = rules_df
+    try:
+        if (
+            isinstance(rules_df, pd.DataFrame)
+            and not rules_df.empty
+            and "RAT" in rules_df.columns
+        ):
+            relevant_rules = rules_df[rules_df["RAT"] == rat].copy()
+    except Exception:  # noqa: BLE001
+        relevant_rules = rules_df
 
     fig = build_drilldown_plot(
         df=d[d["site_code"] == int(code_int)],
@@ -973,12 +975,6 @@ def _update_site_view(event=None) -> None:
         rat=rat,
     )
     trend_plot_pane.object = fig
-
-    rules_df = (
-        rules_table.value
-        if isinstance(rules_table.value, pd.DataFrame)
-        else pd.DataFrame()
-    )
     kpis_for_heatmap = []
     if (
         isinstance(site_df, pd.DataFrame)
@@ -1488,6 +1484,52 @@ def _refresh_filtered_results(event=None) -> None:
         current_top_anomalies_df = pd.DataFrame()
         top_anomalies_table.value = current_top_anomalies_df
 
+    if current_multirat_raw is not None and not current_multirat_raw.empty:
+        cm = _apply_city_filter(current_multirat_raw)
+        if "is_complaint_site" in cm.columns:
+            cm = cm[cm["is_complaint_site"] == True]  # noqa: E712
+        else:
+            cm = cm.iloc[0:0].copy()
+        score_col = (
+            "criticality_score_weighted"
+            if "criticality_score_weighted" in cm.columns
+            else "criticality_score"
+        )
+        if score_col in cm.columns:
+            cm = cm[
+                pd.to_numeric(cm[score_col], errors="coerce").fillna(0)
+                >= int(min_criticality.value)
+            ]
+            cm = cm.sort_values(by=[score_col], ascending=False)
+        complaint_multirat_summary_table.value = cm
+    else:
+        complaint_multirat_summary_table.value = pd.DataFrame()
+
+    if current_top_anomalies_raw is not None and not current_top_anomalies_raw.empty:
+        ct = _apply_city_filter(current_top_anomalies_raw)
+        if "is_complaint_site" in ct.columns:
+            ct = ct[ct["is_complaint_site"] == True]  # noqa: E712
+        else:
+            ct = ct.iloc[0:0].copy()
+        if top_rat_filter.value:
+            ct = ct[ct["RAT"].isin(list(top_rat_filter.value))]
+        if top_status_filter.value and "status" in ct.columns:
+            ct = ct[ct["status"].isin(list(top_status_filter.value))]
+        score_col = (
+            "anomaly_score_weighted"
+            if "anomaly_score_weighted" in ct.columns
+            else "anomaly_score"
+        )
+        if score_col in ct.columns:
+            ct = ct[
+                pd.to_numeric(ct[score_col], errors="coerce").fillna(0)
+                >= int(min_anomaly_score.value)
+            ]
+            ct = ct.sort_values(by=[score_col], ascending=False)
+        complaint_top_anomalies_table.value = ct
+    else:
+        complaint_top_anomalies_table.value = pd.DataFrame()
+
     current_export_bytes = None
 
 
@@ -1833,6 +1875,8 @@ def load_datasets(event=None) -> None:
         site_summary_table.value = pd.DataFrame()
         multirat_summary_table.value = pd.DataFrame()
         top_anomalies_table.value = pd.DataFrame()
+        complaint_multirat_summary_table.value = pd.DataFrame()
+        complaint_top_anomalies_table.value = pd.DataFrame()
         site_kpi_table.value = pd.DataFrame()
         trend_plot_pane.object = None
         heatmap_plot_pane.object = None
@@ -1924,8 +1968,20 @@ def load_datasets(event=None) -> None:
         _loading_datasets = False
         try:
             _update_site_options()
+        except Exception:  # noqa: BLE001
+            pass
+
+        try:
             _update_kpi_options()
+        except Exception:  # noqa: BLE001
+            pass
+
+        try:
             _update_site_view()
+        except Exception:  # noqa: BLE001
+            pass
+
+        try:
             _refresh_validation_state()
         except Exception:  # noqa: BLE001
             pass
@@ -2088,6 +2144,16 @@ def _build_export_bytes() -> bytes:
             if isinstance(current_top_anomalies_df, pd.DataFrame)
             else None
         ),
+        (
+            complaint_multirat_summary_table.value
+            if isinstance(complaint_multirat_summary_table.value, pd.DataFrame)
+            else None
+        ),
+        (
+            complaint_top_anomalies_table.value
+            if isinstance(complaint_top_anomalies_table.value, pd.DataFrame)
+            else None
+        ),
     )
 
 
@@ -2176,6 +2242,13 @@ try:
 except Exception:  # noqa: BLE001
     pass
 
+try:
+    complaint_top_anomalies_table.on_click(
+        lambda e: _handle_double_click("top", complaint_top_anomalies_table, e)
+    )
+except Exception:  # noqa: BLE001
+    pass
+
 try:
     multirat_summary_table.on_click(
         lambda e: _handle_double_click("multirat", multirat_summary_table, e)
@@ -2183,6 +2256,13 @@ try:
 except Exception:  # noqa: BLE001
     pass
 
+try:
+    complaint_multirat_summary_table.on_click(
+        lambda e: _handle_double_click("multirat", complaint_multirat_summary_table, e)
+    )
+except Exception:  # noqa: BLE001
+    pass
+
 min_criticality.param.watch(_refresh_filtered_results, "value")
 min_anomaly_score.param.watch(_refresh_filtered_results, "value")
 city_filter.param.watch(_refresh_filtered_results, "value")
@@ -2228,7 +2308,7 @@ def _build_drilldown_export_bytes() -> bytes:
         if kpi_select.value and str(kpi_select.value) not in selected_kpis:
             selected_kpis = [str(kpi_select.value)] + selected_kpis
 
-    selected_kpis = [k for k in selected_kpis if k in s.columns]
+    selected_kpis = [k for k in selected_kpis if k in d.columns]
     base_cols = ["date_only"]
     daily_cols = base_cols + selected_kpis
     daily_out = s[daily_cols].copy() if selected_kpis else s[["date_only"]].copy()
@@ -2243,6 +2323,7 @@ def _build_drilldown_export_bytes() -> bytes:
         summary_out = pd.DataFrame()
     else:
         baseline_start, baseline_end, recent_start, recent_end = windows
+
         rows = []
         for k in selected_kpis:
             rule = _infer_rule_row(rules_df, str(rat), str(k))
@@ -2275,31 +2356,42 @@ def _build_drilldown_export_bytes() -> bytes:
 
             bad_flags = []
             recent_vals = sk.loc[recent_mask, ["date_only", k]].sort_values("date_only")
+            bad_dates: list[date] = []
             for _, r in recent_vals.iterrows():
                 v = r.get(k)
-                bad_flags.append(
-                    bool(
-                        is_bad(
-                            float(v) if pd.notna(v) else None,
-                            baseline_med,
-                            direction,
-                            float(rel_threshold_pct.value),
-                            sla_val,
-                        )
+                is_bad_day = bool(
+                    is_bad(
+                        float(v) if pd.notna(v) else None,
+                        baseline_med,
+                        direction,
+                        float(rel_threshold_pct.value),
+                        sla_val,
                     )
                 )
+                bad_flags.append(is_bad_day)
+                if is_bad_day:
+                    try:
+                        d0 = r.get("date_only")
+                        if d0 is not None:
+                            bad_dates.append(
+                                d0
+                                if isinstance(d0, date)
+                                else pd.to_datetime(d0).date()
+                            )
+                    except Exception:  # noqa: BLE001
+                        pass
 
             rows.append(
                 {
                     "RAT": str(rat),
-                    "site_code": int(code),
+                    "site_code": int(code_int),
                     "KPI": str(k),
                     "direction": direction,
                     "sla": sla_val,
                     "baseline_median": baseline_med,
                     "recent_median": recent_med,
                     "bad_days_recent": int(sum(bad_flags)),
-                    "max_streak_recent": int(max_consecutive_days(bad_flags)),
+                    "max_streak_recent": int(max_consecutive_days(bad_dates)),
                 }
             )
         summary_out = pd.DataFrame(rows)
@@ -2334,7 +2426,6 @@ def _drilldown_export_callback() -> io.BytesIO:
 drilldown_export_button.callback = _drilldown_export_callback
 
 
-# Page layout components (used by the multipage portal)
 sidebar = pn.Column(
     file_2g,
     file_3g,
@@ -2373,9 +2464,7 @@ sidebar = pn.Column(
     export_button,
 )
 
-main = pn.Column(
-    status_pane,
-    validation_pane,
+_tab_overview = pn.Column(
     pn.pane.Markdown("## Datasets"),
     datasets_table,
     pn.pane.Markdown("## KPI Rules (editable)"),
@@ -2386,7 +2475,19 @@ main = pn.Column(
     multirat_summary_table,
     pn.pane.Markdown("## Top anomalies (cross-RAT)"),
     top_anomalies_table,
-    pn.layout.Divider(),
+    sizing_mode="stretch_width",
+)
+
+_tab_complaint = pn.Column(
+    pn.pane.Markdown("## Complaint sites only"),
+    pn.pane.Markdown("### Multi-RAT Summary (Complaint)"),
+    complaint_multirat_summary_table,
+    pn.pane.Markdown("### Top anomalies (Complaint)"),
+    complaint_top_anomalies_table,
+    sizing_mode="stretch_width",
+)
+
+_tab_drilldown = pn.Column(
     pn.pane.Markdown("## Drill-down"),
     pn.Row(site_select, rat_select),
     pn.Row(kpi_group_select, kpi_group_mode),
@@ -2395,6 +2496,21 @@ main = pn.Column(
     pn.Column(trend_plot_pane, sizing_mode="stretch_both", min_height=500),
     pn.Column(heatmap_plot_pane, sizing_mode="stretch_both", min_height=400),
     pn.Column(hist_plot_pane, sizing_mode="stretch_both", min_height=400),
+    sizing_mode="stretch_both",
+)
+
+_tabs_main = pn.Tabs(
+    ("Overview", _tab_overview),
+    ("Complaint sites only", _tab_complaint),
+    ("Drill-down", _tab_drilldown),
+    dynamic=True,
+    sizing_mode="stretch_both",
+)
+
+main = pn.Column(
+    status_pane,
+    validation_pane,
+    _tabs_main,
 )
 
 

process_kpi/kpi_health_check/export.py

@@ -10,6 +10,8 @@ def build_export_bytes(
     status_df: pd.DataFrame | None,
     multirat_summary_df: pd.DataFrame | None = None,
     top_anomalies_df: pd.DataFrame | None = None,
+    complaint_multirat_df: pd.DataFrame | None = None,
+    complaint_top_anomalies_df: pd.DataFrame | None = None,
 ) -> bytes:
     dfs = [
         datasets_df if isinstance(datasets_df, pd.DataFrame) else pd.DataFrame(),
@@ -26,6 +28,16 @@ def build_export_bytes(
             if isinstance(top_anomalies_df, pd.DataFrame)
             else pd.DataFrame()
         ),
+        (
+            complaint_multirat_df
+            if isinstance(complaint_multirat_df, pd.DataFrame)
+            else pd.DataFrame()
+        ),
+        (
+            complaint_top_anomalies_df
+            if isinstance(complaint_top_anomalies_df, pd.DataFrame)
+            else pd.DataFrame()
+        ),
     ]
     sheet_names = [
         "Datasets",
@@ -34,5 +46,7 @@ def build_export_bytes(
         "Site_KPI_Status",
         "MultiRAT_Summary",
         "Top_Anomalies",
+        "Complaint_MultiRAT",
+        "Complaint_Top_Anomalies",
     ]
     return write_dfs_to_excel(dfs, sheet_names, index=False)