feat: Introduce KPI health check feature with panel application, drilldown plots, and rule presets.

panel_app/kpi_health_check_drilldown_plots.py

@@ -0,0 +1,105 @@
+import pandas as pd
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+
+def build_drilldown_plot(
+    df: pd.DataFrame,
+    kpis: list[str],
+    rules_df: pd.DataFrame | None = None,
+    highlight_bad_days: bool = True,
+    show_sla: bool = True,
+    site_code: str | int = "",
+    rat: str = "",
+) -> go.Figure | None:
+    """
+    Builds the drill-down trend plot with native Plotly annotations.
+    """
+    if df is None or df.empty or not kpis:
+        return None
+
+    # Filter columns that exist
+    valid_kpis = [k for k in kpis if k in df.columns]
+    if not valid_kpis:
+        return None
+
+    # Sort by date
+    # Sort by date
+    plot_df = df.sort_values("date_only").copy()
+    
+    title_text = f"{rat} - Site {site_code}"
+    # If single KPI, be explicit in title
+    if len(valid_kpis) == 1:
+        title_text = f"{rat} - Site {site_code} - {valid_kpis[0]}"
+
+    # Subplot for Timeline (Streak) - Row 2
+    # Row 1: Main Trend
+    fig = make_subplots(
+        rows=2, cols=1, 
+        shared_xaxes=True, 
+        vertical_spacing=0.05,
+        row_heights=[0.85, 0.15],
+        subplot_titles=(title_text, "Status Check")
+    )
+
+    for kpi in valid_kpis:
+        # Data preparation
+        x_data = plot_df["date_only"]
+        y_data = pd.to_numeric(plot_df[kpi], errors="coerce")
+        
+        # Add Trace
+        fig.add_trace(
+            go.Scatter(
+                x=x_data,
+                y=y_data,
+                mode="lines+markers",
+                name=kpi,
+                legendgroup=kpi,  # Allows grouping logic if needed
+            ),
+            row=1, col=1
+        )
+
+        # Add SLA line if available
+        if show_sla and rules_df is not None:
+            try:
+                # Find rule for this KPI
+                # Note: This implies rules_df is filtered for the correct RAT
+                rule = rules_df[rules_df["KPI"] == kpi]
+                if not rule.empty:
+                    sla_val = pd.to_numeric(rule.iloc[0].get("sla"), errors="coerce")
+                    if pd.notna(sla_val):
+                        fig.add_hline(
+                            y=sla_val, 
+                            line_dash="dot", 
+                            line_color="red", 
+                            annotation_text=f"SLA {kpi}",
+                            annotation_position="bottom right",
+                            row=1, col=1
+                        )
+            except Exception:
+                pass
+
+    # Timeline / Streak Subplot
+    # We add a dummy invisible trace to ensure the X-axis (shared) renders dates at the bottom
+    # even if we haven't computed 'is_bad' explicitly here yet.
+    # This addresses "je veux voir les dates sur l'axe en bas".
+    if not plot_df.empty:
+        fig.add_trace(
+            go.Scatter(
+                x=plot_df["date_only"],
+                y=[0] * len(plot_df),
+                mode="markers",
+                opacity=0,
+                showlegend=False,
+                hoverinfo="skip"
+            ),
+            row=2, col=1
+        )
+    
+    fig.update_layout(
+        template="plotly_white",
+        height=500,
+        margin=dict(l=50, r=50, t=50, b=50),
+        legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1)
+    )
+    
+    return fig

panel_app/kpi_health_check_panel.py

@@ -43,6 +43,9 @@ from process_kpi.kpi_health_check.profiles import (
     save_profile,
 )
 from process_kpi.kpi_health_check.rules import infer_kpi_direction, infer_kpi_sla
+from process_kpi.kpi_health_check.kpi_groups import get_kpis_by_group, filter_kpis
+from process_kpi.kpi_health_check.benchmarks import calculate_sla_metrics
+from panel_app.kpi_health_check_drilldown_plots import build_drilldown_plot
 
 pn.extension("plotly", "tabulator")
 
@@ -161,6 +164,9 @@ def _drilldown_cache_key(site_code: int, rat: str, kpi: str) -> tuple:
         int(recent_days.value),
         float(rel_threshold_pct.value),
         int(min_consecutive_days.value),
+        # New cache keys
+        str(kpi_group_select.value),
+        str(kpi_group_mode.value),
     )
 
 
@@ -400,6 +406,16 @@ kpi_compare_select = pn.widgets.MultiChoice(name="Compare KPIs", options=[], val
 kpi_compare_norm = pn.widgets.Select(
     name="Normalization", options=["None", "Min-Max", "Z-score"], value="None"
 )
+
+# NEW WIDGETS
+kpi_group_select = pn.widgets.Select(
+    name="KPI Group", options=["All (selected KPIs)"], value="All (selected KPIs)"
+)
+kpi_group_mode = pn.widgets.Select(
+    name="Group Mode", 
+    options=["Filter KPI list only (recommended)", "Add top 12 KPIs to compare"],
+    value="Filter KPI list only (recommended)"
+)
 drilldown_export_button = pn.widgets.FileDownload(
     label="Download drill-down",
     filename="KPI_Drilldown.xlsx",
@@ -407,7 +423,7 @@ drilldown_export_button = pn.widgets.FileDownload(
 )
 
 site_kpi_table = pn.widgets.Tabulator(
-    height=260, sizing_mode="stretch_width", layout="fit_data_table"
+    height=520, sizing_mode="stretch_width", layout="fit_data_table"
 )
 
 _set_tabulator_pagination(site_summary_table, page_size=50)
@@ -496,11 +512,29 @@ def _update_kpi_options() -> None:
         if c not in {"site_code", "date_only", "Longitude", "Latitude", "City", "RAT"}
     ]
     kpis = sorted([str(c) for c in kpis])
+
+    # Apply Grouping if needed
+    groups = get_kpis_by_group(kpis)
+    group_options = ["All (selected KPIs)"] + sorted([g for g in groups.keys() if g != "Other"])
+    if "Other" in groups:
+        group_options.append("Other")
+    
     _updating_drilldown = True
     try:
-        kpi_select.options = kpis
-        if kpis and kpi_select.value not in kpis:
-            kpi_select.value = kpis[0]
+        kpi_group_select.options = group_options
+        if kpi_group_select.value not in group_options:
+            kpi_group_select.value = group_options[0]
+
+        # Filter KPIs based on group
+        filtered_kpis = filter_kpis(
+            kpis, 
+            kpi_group_select.value, 
+            mode=kpi_group_mode.value
+        )
+        
+        kpi_select.options = filtered_kpis
+        if filtered_kpis and kpi_select.value not in filtered_kpis:
+            kpi_select.value = filtered_kpis[0]
 
         kpi_compare_select.options = kpis
         cur = list(kpi_compare_select.value or [])
@@ -563,15 +597,40 @@ def _update_site_view(event=None) -> None:
         )
     except Exception:  # noqa: BLE001
         available_sites = set()
+
+    
+    # Robustly resolve code_int from site_select.value
+    # AutocompleteInput might return the Label (str) or Value (int) depending on usage
+    code_int = None
+    if code is not None:
+        # 1. Try if code is already a known value (int)
+        if hasattr(site_select, "options") and isinstance(site_select.options, dict):
+            # Check if it matches a Key (Label)
+            if code in site_select.options:
+                code_int = site_select.options[code]
+            # Check if it is a Value in the dict
+            elif code in site_select.options.values():
+                code_int = code
+        
+        # 2. If not checking opts or not found, try direct cast
+        if code_int is None:
+            try:
+                code_int = int(code)
+            except Exception:
+                code_int = None
+
     if available_sites:
-        try:
-            code_int = int(code)
-        except Exception:  # noqa: BLE001
-            code_int = None
         if code_int is None or code_int not in available_sites:
             new_code = next(iter(sorted(available_sites)))
+            # We must set the WIDGET to a value it accepts. 
+            # If options are {Label: ID}, and we set ID, it usually works.
             _set_widget_value(site_select, new_code)
             code = new_code
+            code_int = new_code
+        else:
+            # If valid, ensure we stick with the int representation for downstream logic
+            # but don't force widget update if not needed prevents loops
+            pass
 
     status_df = (
         current_status_df
@@ -582,7 +641,7 @@ def _update_site_view(event=None) -> None:
         site_df = pd.DataFrame()
     else:
         site_df = status_df[
-            (status_df["site_code"] == int(code)) & (status_df["RAT"] == rat)
+            (status_df["site_code"] == int(code_int if code_int is not None else 0)) & (status_df["RAT"] == rat)
         ].copy()
     site_kpi_table.value = site_df
 
@@ -615,47 +674,50 @@ def _update_site_view(event=None) -> None:
         trend_plot_pane.object, heatmap_plot_pane.object, hist_plot_pane.object = cached
         return
 
-    selected_kpis = [str(x) for x in (kpi_compare_select.value or []) if str(x)]
-    if not selected_kpis:
-        selected_kpis = [str(kpi)] if kpi else []
-    else:
-        if kpi and str(kpi) not in selected_kpis:
-            selected_kpis = [str(kpi)] + selected_kpis
-
-    selected_kpis = [x for x in selected_kpis if x in s.columns]
-    if not selected_kpis:
-        trend_plot_pane.object = None
-    else:
-        plot_df = s[["date_only"] + selected_kpis].copy()
-        for col in selected_kpis:
-            plot_df[col] = pd.to_numeric(plot_df[col], errors="coerce")
-
-        norm_mode = str(kpi_compare_norm.value or "None")
-        if norm_mode != "None":
-            for col in selected_kpis:
-                vals = pd.to_numeric(plot_df[col], errors="coerce")
-                if norm_mode == "Min-Max":
-                    vmin = float(vals.min()) if vals.notna().any() else 0.0
-                    vmax = float(vals.max()) if vals.notna().any() else 0.0
-                    denom = vmax - vmin
-                    plot_df[col] = (vals - vmin) / denom if denom else 0.0
-                elif norm_mode == "Z-score":
-                    mean = float(vals.mean()) if vals.notna().any() else 0.0
-                    std = float(vals.std()) if vals.notna().any() else 0.0
-                    plot_df[col] = (vals - mean) / std if std else 0.0
-
-        df_long = plot_df.melt(
-            id_vars=["date_only"],
-            value_vars=selected_kpis,
-            var_name="KPI",
-            value_name="value",
+    # Determine KPIs to plot based on group mode
+    kpis_to_plot = []
+    
+    # 1. Start with explicitly selected 'Compare KPIs'
+    selected = [str(x) for x in (kpi_compare_select.value or []) if str(x)]
+    
+    # 2. Add the primary selected KPI if not present
+    if kpi and str(kpi) not in selected:
+        selected = [str(kpi)] + selected
+    
+    # 3. Handle Group Mode "Add top 12 KPIs"
+    # If mode is "Add top...", we fetch from group and append
+    if "Add top" in str(kpi_group_mode.value):
+        from_group = filter_kpis(
+            d.columns.tolist(), 
+            kpi_group_select.value, 
+            mode="Top-N",
+            top_n=12
         )
-        title = f"{rat} - site {int(code)}"
-        if norm_mode != "None" and len(selected_kpis) > 1:
-            title = f"{title} (compare, {norm_mode})"
-        fig = px.line(df_long, x="date_only", y="value", color="KPI", markers=True)
-        fig.update_layout(template="plotly_white", title=title)
-        trend_plot_pane.object = fig
+        # Merge unique
+        for gk in from_group:
+            if gk not in selected:
+                selected.append(gk)
+                
+    # Safeguard: Limit to 15 max to prevent browser crash
+    kpis_to_plot = selected[:15]
+
+    # Build Plot using new module
+    # We need the rules for this RAT/KPIs to show SLA
+    relevant_rules = pd.DataFrame()
+    if isinstance(rules_table.value, pd.DataFrame) and not rules_table.value.empty:
+        r = rules_table.value
+        relevant_rules = r[r["RAT"] == rat]
+
+    fig = build_drilldown_plot(
+        df=d[d["site_code"] == int(code)],
+        kpis=kpis_to_plot,
+        rules_df=relevant_rules,
+        highlight_bad_days=True,
+        show_sla=True,
+        site_code=code,
+        rat=rat
+    )
+    trend_plot_pane.object = fig
 
     rules_df = (
         rules_table.value
@@ -1772,6 +1834,8 @@ def _on_drilldown_change(event=None) -> None:
 
 
 rat_select.param.watch(_on_rat_change, "value")
+kpi_group_select.param.watch(_on_rat_change, "value")  # Updating group also needs to re-filter KPI options
+kpi_group_mode.param.watch(_on_drilldown_change, "value")
 site_select.param.watch(_on_drilldown_change, "value")
 kpi_select.param.watch(_on_drilldown_change, "value")
 kpi_compare_select.param.watch(_on_drilldown_change, "value")
@@ -1997,15 +2061,12 @@ main = pn.Column(
     pn.layout.Divider(),
     pn.pane.Markdown("## Drill-down"),
     pn.Row(site_select, rat_select),
+    pn.Row(kpi_group_select, kpi_group_mode),
     pn.Row(kpi_select, kpi_compare_select, kpi_compare_norm, drilldown_export_button),
-    pn.Row(
-        pn.Column(site_kpi_table, sizing_mode="stretch_width"),
-        pn.Column(trend_plot_pane, sizing_mode="stretch_both"),
-    ),
-    pn.Row(
-        pn.Column(heatmap_plot_pane, sizing_mode="stretch_both"),
-        pn.Column(hist_plot_pane, sizing_mode="stretch_both"),
-    ),
+    pn.Column(site_kpi_table, sizing_mode="stretch_width"),
+    pn.Column(trend_plot_pane, sizing_mode="stretch_both", min_height=500),
+    pn.Column(heatmap_plot_pane, sizing_mode="stretch_both", min_height=400),
+    pn.Column(hist_plot_pane, sizing_mode="stretch_both", min_height=400),
 )
 
 

process_kpi/kpi_health_check/benchmarks.py

@@ -0,0 +1,42 @@
+import pandas as pd
+import numpy as np
+
+def calculate_sla_metrics(
+    df: pd.DataFrame, 
+    kpi: str, 
+    rules_df: pd.DataFrame | None = None
+) -> dict:
+    """
+    Calculates simple metrics for the given KPI trace:
+    - SLA value (if exists)
+    - Median (recent window)
+    
+    Returns a dict with: 'sla': float|None, 'median': float|None
+    """
+    res = {"sla": None, "median": None}
+    
+    if df is None or df.empty or kpi not in df.columns:
+        return res
+
+    # 1. Get SLA from rules
+    if rules_df is not None and not rules_df.empty:
+        # Assuming rules_df has 'KPI' and 'sla' columns
+        # We also need to match RAT? usually passed or handled outside.
+        # Here we do a simplistic lookup.
+        try:
+            row = rules_df[rules_df["KPI"] == kpi]
+            if not row.empty:
+                val = row.iloc[0].get("sla")
+                res["sla"] = float(val) if pd.notna(val) else None
+        except Exception:
+            pass
+
+    # 2. Calculate Median (entire passed df, usually it's the recent window)
+    try:
+        vals = pd.to_numeric(df[kpi], errors="coerce").dropna()
+        if not vals.empty:
+            res["median"] = float(vals.median())
+    except Exception:
+        pass
+        
+    return res

process_kpi/kpi_health_check/kpi_groups.py

@@ -0,0 +1,89 @@
+import re
+import pandas as pd
+
+# Regex patterns for KPI classification
+# Order matters: first match wins
+PATTERNS = {
+    # Refined patterns based on user data
+    "Success Rate": r"(?i)(cssr|success|attach|setup|establ|answer|complete|connected|ho.*succ|\berab\b|\brrc\b.*(?:\bsr\b|rate|succ)|\basr\b|\bsr\b)",
+    "Fails/Drop/Block": r"(?i)(drop|dcr|fail|block|reject|deny|loss|lost|discard|congestion|accessibility.*fail|retention.*fail)",
+    "Throughput": r"(?i)(throughput|thp|thrput|debit|dl.*rate|ul.*rate|bitrate)",
+    "Traffic": r"(?i)(traffic|volume|erl|payload|gbytes|gb|load|usage|utilization|users|subscribers|ue|conn.*ue)",
+    "Availability": r"(?i)(availability|avail|unavailability|unavail|dispo|disponibil|uptime)",
+    "Latency": r"(?i)(latency|delay|\brt\b|rtt)",
+}
+
+def classify_kpi(kpi_name: str) -> str:
+    """
+    Classifies a KPI name into a group based on regex patterns.
+    Returns 'Other' if no match found.
+    """
+    kpi_str = str(kpi_name)
+    for group, pattern in PATTERNS.items():
+        if re.search(pattern, kpi_str):
+            return group
+    return "Other"
+
+def get_kpis_by_group(all_kpis: list[str]) -> dict[str, list[str]]:
+    """
+    Returns a dictionary mapping group names to lists of KPIs.
+    """
+    groups = {g: [] for g in PATTERNS.keys()}
+    groups["Other"] = []
+    
+    for kpi in sorted(all_kpis):
+        group = classify_kpi(kpi)
+        groups[group].append(kpi)
+        
+    # Remove empty groups
+    return {k: v for k, v in groups.items() if v}
+
+def filter_kpis(
+    all_kpis: list[str], 
+    group: str, 
+    mode: str = "Filter", 
+    top_n: int = 12,
+    stats_df: pd.DataFrame | None = None
+) -> list[str]:
+    """
+    Filters KPIs based on the selected group and mode.
+    
+    Args:
+        all_kpis: List of available KPI names.
+        group: Selected group name (or 'All').
+        mode: 'Filter' or 'Top-N'.
+        top_n: Max KPIs to return if filtering needs truncation or specific selection.
+        stats_df: Optional DataFrame with 'site_code', 'KPI', 'is_bad', etc. for sorting.
+    """
+    if not all_kpis:
+        return []
+
+    # 1. Filter by group
+    if group and group != "All (selected KPIs)":
+        # Handle "Success Rate (>= SLA...)" formatted names if passed from UI
+        clean_group = group.split(" (")[0]
+        # Basic mapping check - if the group name in UI has extra text, we match key prefix
+        target_group = "Other"
+        for k in PATTERNS.keys():
+            if k in group:
+                target_group = k
+                break
+        if "Other" in group:
+            target_group = "Other"
+            
+        candidates = [k for k in all_kpis if classify_kpi(k) == target_group]
+    else:
+        candidates = list(all_kpis)
+
+    if not candidates:
+        return []
+
+    # 2. Sort/Limit if needed
+    # If we have stats, we can sort by "badness" or variance
+    # For now, simplistic alpha sort unless we have stats
+    if stats_df is not None and not stats_df.empty:
+        # TODO: Implement smart sorting based on stats if available
+        # For V1, we just return candidates sorted alphabetically
+        pass
+
+    return sorted(candidates)