Add 3G and LTE cell availability analysis with SLA comparison, multi-RAT site-level reporting, and export functionality

apps/kpi_analysis/trafic_analysis.py

@@ -98,9 +98,24 @@ def preprocess_3g(df: pd.DataFrame) -> pd.DataFrame:
         },
         inplace=True,
     )
-    df = df.groupby(["date", "ID", "code"], as_index=False)[
-        ["3g_voice_trafic", "3g_data_trafic"]
-    ].sum()
+
+    kpi_col = None
+    for col in df.columns:
+        if "cell availability" in str(col).lower():
+            kpi_col = col
+            break
+
+    if kpi_col is not None:
+        df["3g_cell_avail"] = pd.to_numeric(df[kpi_col], errors="coerce")
+
+    agg_dict = {
+        "3g_voice_trafic": "sum",
+        "3g_data_trafic": "sum",
+    }
+    if "3g_cell_avail" in df.columns:
+        agg_dict["3g_cell_avail"] = "mean"
+
+    df = df.groupby(["date", "ID", "code"], as_index=False).agg(agg_dict)
     return df
 
 
@@ -119,7 +134,14 @@ def preprocess_lte(df: pd.DataFrame) -> pd.DataFrame:
     )
     df["date"] = pd.to_datetime(df["PERIOD_START_TIME"], format=date_format)
     df["ID"] = df["date"].astype(str) + "_" + df["code"].astype(str)
-    df = df.groupby(["date", "ID", "code"], as_index=False)[["lte_data_trafic"]].sum()
+    if "Cell Avail excl BLU" in df.columns:
+        df["lte_cell_avail"] = pd.to_numeric(df["Cell Avail excl BLU"], errors="coerce")
+
+    agg_dict = {"lte_data_trafic": "sum"}
+    if "lte_cell_avail" in df.columns:
+        agg_dict["lte_cell_avail"] = "mean"
+
+    df = df.groupby(["date", "ID", "code"], as_index=False).agg(agg_dict)
     return df
 
 
@@ -149,14 +171,15 @@ def merge_and_compare(df_2g, df_3g, df_lte, pre_range, post_range, last_period_r
         if col not in df:
             df[col] = 0
 
-    tch_mask = None
-    if "2g_tch_avail" in df.columns:
-        tch_mask = df["2g_tch_avail"].notna()
+    kpi_masks = {}
+    for kpi_col in ["2g_tch_avail", "3g_cell_avail", "lte_cell_avail"]:
+        if kpi_col in df.columns:
+            kpi_masks[kpi_col] = df[kpi_col].notna()
 
     df.fillna(0, inplace=True)
 
-    if tch_mask is not None:
-        df.loc[~tch_mask, "2g_tch_avail"] = np.nan
+    for kpi_col, mask in kpi_masks.items():
+        df.loc[~mask, kpi_col] = np.nan
 
     df["total_voice_trafic"] = df["2g_voice_trafic"] + df["3g_voice_trafic"]
     df["total_data_trafic"] = (
@@ -366,6 +389,266 @@ def analyze_2g_availability(df: pd.DataFrame, sla_2g: float):
     return summary_df, site_pivot
 
 
+def analyze_3g_availability(df: pd.DataFrame, sla_3g: float):
+    avail_col = "3g_cell_avail"
+
+    if avail_col not in df.columns or "period" not in df.columns:
+        return None, None
+
+    df_3g = df[df[avail_col].notna()].copy()
+    df_3g = df_3g[df_3g["period"].isin(["pre", "post"])]
+
+    if df_3g.empty:
+        return None, None
+
+    site_pivot = df_3g.groupby(["code", "period"])[avail_col].mean().unstack()
+
+    site_pivot = site_pivot.rename(
+        columns={"pre": "cell_avail_pre", "post": "cell_avail_post"}
+    )
+
+    if "cell_avail_pre" not in site_pivot.columns:
+        site_pivot["cell_avail_pre"] = pd.NA
+    if "cell_avail_post" not in site_pivot.columns:
+        site_pivot["cell_avail_post"] = pd.NA
+
+    site_pivot["cell_avail_diff"] = (
+        site_pivot["cell_avail_post"] - site_pivot["cell_avail_pre"]
+    )
+    site_pivot["pre_ok_vs_sla"] = site_pivot["cell_avail_pre"] >= sla_3g
+    site_pivot["post_ok_vs_sla"] = site_pivot["cell_avail_post"] >= sla_3g
+
+    site_pivot = site_pivot.reset_index()
+
+    summary_rows = []
+    for period_label, col_name in [
+        ("pre", "cell_avail_pre"),
+        ("post", "cell_avail_post"),
+    ]:
+        series = site_pivot[col_name].dropna()
+        total_cells = series.shape[0]
+        if total_cells == 0:
+            summary_rows.append(
+                {
+                    "period": period_label,
+                    "cells": 0,
+                    "avg_availability": pd.NA,
+                    "median_availability": pd.NA,
+                    "p05_availability": pd.NA,
+                    "p95_availability": pd.NA,
+                    "min_availability": pd.NA,
+                    "max_availability": pd.NA,
+                    "cells_ge_sla": 0,
+                    "cells_lt_sla": 0,
+                    "pct_cells_ge_sla": pd.NA,
+                }
+            )
+            continue
+        cells_ge_sla = (series >= sla_3g).sum()
+        cells_lt_sla = (series < sla_3g).sum()
+        summary_rows.append(
+            {
+                "period": period_label,
+                "cells": int(total_cells),
+                "avg_availability": series.mean(),
+                "median_availability": series.median(),
+                "p05_availability": series.quantile(0.05),
+                "p95_availability": series.quantile(0.95),
+                "min_availability": series.min(),
+                "max_availability": series.max(),
+                "cells_ge_sla": int(cells_ge_sla),
+                "cells_lt_sla": int(cells_lt_sla),
+                "pct_cells_ge_sla": cells_ge_sla / total_cells * 100,
+            }
+        )
+
+    summary_df = pd.DataFrame(summary_rows)
+
+    return summary_df, site_pivot
+
+
+def analyze_lte_availability(df: pd.DataFrame, sla_lte: float):
+    avail_col = "lte_cell_avail"
+
+    if avail_col not in df.columns or "period" not in df.columns:
+        return None, None
+
+    df_lte = df[df[avail_col].notna()].copy()
+    df_lte = df_lte[df_lte["period"].isin(["pre", "post"])]
+
+    if df_lte.empty:
+        return None, None
+
+    site_pivot = df_lte.groupby(["code", "period"])[avail_col].mean().unstack()
+
+    site_pivot = site_pivot.rename(
+        columns={"pre": "lte_avail_pre", "post": "lte_avail_post"}
+    )
+
+    if "lte_avail_pre" not in site_pivot.columns:
+        site_pivot["lte_avail_pre"] = pd.NA
+    if "lte_avail_post" not in site_pivot.columns:
+        site_pivot["lte_avail_post"] = pd.NA
+
+    site_pivot["lte_avail_diff"] = (
+        site_pivot["lte_avail_post"] - site_pivot["lte_avail_pre"]
+    )
+    site_pivot["pre_ok_vs_sla"] = site_pivot["lte_avail_pre"] >= sla_lte
+    site_pivot["post_ok_vs_sla"] = site_pivot["lte_avail_post"] >= sla_lte
+
+    site_pivot = site_pivot.reset_index()
+
+    summary_rows = []
+    for period_label, col_name in [
+        ("pre", "lte_avail_pre"),
+        ("post", "lte_avail_post"),
+    ]:
+        series = site_pivot[col_name].dropna()
+        total_cells = series.shape[0]
+        if total_cells == 0:
+            summary_rows.append(
+                {
+                    "period": period_label,
+                    "cells": 0,
+                    "avg_availability": pd.NA,
+                    "median_availability": pd.NA,
+                    "p05_availability": pd.NA,
+                    "p95_availability": pd.NA,
+                    "min_availability": pd.NA,
+                    "max_availability": pd.NA,
+                    "cells_ge_sla": 0,
+                    "cells_lt_sla": 0,
+                    "pct_cells_ge_sla": pd.NA,
+                }
+            )
+            continue
+        cells_ge_sla = (series >= sla_lte).sum()
+        cells_lt_sla = (series < sla_lte).sum()
+        summary_rows.append(
+            {
+                "period": period_label,
+                "cells": int(total_cells),
+                "avg_availability": series.mean(),
+                "median_availability": series.median(),
+                "p05_availability": series.quantile(0.05),
+                "p95_availability": series.quantile(0.95),
+                "min_availability": series.min(),
+                "max_availability": series.max(),
+                "cells_ge_sla": int(cells_ge_sla),
+                "cells_lt_sla": int(cells_lt_sla),
+                "pct_cells_ge_sla": cells_ge_sla / total_cells * 100,
+            }
+        )
+
+    summary_df = pd.DataFrame(summary_rows)
+
+    return summary_df, site_pivot
+
+
+def analyze_multirat_availability(
+    df: pd.DataFrame, sla_2g: float, sla_3g: float, sla_lte: float
+):
+    if "period" not in df.columns:
+        return None
+
+    rat_cols = []
+    if "2g_tch_avail" in df.columns:
+        rat_cols.append("2g_tch_avail")
+    if "3g_cell_avail" in df.columns:
+        rat_cols.append("3g_cell_avail")
+    if "lte_cell_avail" in df.columns:
+        rat_cols.append("lte_cell_avail")
+
+    if not rat_cols:
+        return None
+
+    agg_dict = {col: "mean" for col in rat_cols}
+
+    pre = df[df["period"] == "pre"].groupby("code", as_index=False).agg(agg_dict)
+    post = df[df["period"] == "post"].groupby("code", as_index=False).agg(agg_dict)
+
+    rename_map_pre = {
+        "2g_tch_avail": "2g_avail_pre",
+        "3g_cell_avail": "3g_avail_pre",
+        "lte_cell_avail": "lte_avail_pre",
+    }
+    rename_map_post = {
+        "2g_tch_avail": "2g_avail_post",
+        "3g_cell_avail": "3g_avail_post",
+        "lte_cell_avail": "lte_avail_post",
+    }
+
+    pre = pre.rename(columns=rename_map_pre)
+    post = post.rename(columns=rename_map_post)
+
+    multi = pd.merge(pre, post, on="code", how="outer")
+
+    if "City" in df.columns:
+        city_df = df[["code", "City"]].drop_duplicates("code")
+        multi = pd.merge(multi, city_df, on="code", how="left")
+
+    # Compute OK/Not OK flags vs SLA on post-period
+    def _ok_flag(series: pd.Series, sla: float) -> pd.Series:
+        if series.name not in multi.columns:
+            return pd.Series([pd.NA] * len(multi), index=multi.index)
+        ok = multi[series.name] >= sla
+        ok = ok.where(multi[series.name].notna(), pd.NA)
+        return ok
+
+    if "2g_avail_post" in multi.columns:
+        multi["ok_2g_post"] = _ok_flag(multi["2g_avail_post"], sla_2g)
+    if "3g_avail_post" in multi.columns:
+        multi["ok_3g_post"] = _ok_flag(multi["3g_avail_post"], sla_3g)
+    if "lte_avail_post" in multi.columns:
+        multi["ok_lte_post"] = _ok_flag(multi["lte_avail_post"], sla_lte)
+
+    def classify_row(row):
+        rats_status = []
+        for rat, col in [
+            ("2G", "ok_2g_post"),
+            ("3G", "ok_3g_post"),
+            ("LTE", "ok_lte_post"),
+        ]:
+            if col in row and not pd.isna(row[col]):
+                rats_status.append((rat, bool(row[col])))
+
+        if not rats_status:
+            return "No RAT data"
+
+        bad_rats = [rat for rat, ok in rats_status if not ok]
+        if not bad_rats:
+            return "OK all RAT"
+        if len(bad_rats) == 1:
+            return f"Degraded {bad_rats[0]} only"
+        return "Degraded multi-RAT (" + ",".join(bad_rats) + ")"
+
+    multi["post_multirat_status"] = multi.apply(classify_row, axis=1)
+
+    # Order columns for readability
+    ordered_cols = ["code"]
+    if "City" in multi.columns:
+        ordered_cols.append("City")
+    for col in [
+        "2g_avail_pre",
+        "2g_avail_post",
+        "3g_avail_pre",
+        "3g_avail_post",
+        "lte_avail_pre",
+        "lte_avail_post",
+        "ok_2g_post",
+        "ok_3g_post",
+        "ok_lte_post",
+        "post_multirat_status",
+    ]:
+        if col in multi.columns:
+            ordered_cols.append(col)
+
+    remaining_cols = [c for c in multi.columns if c not in ordered_cols]
+    multi = multi[ordered_cols + remaining_cols]
+
+    return multi
+
+
 def monthly_data_analysis(df: pd.DataFrame) -> pd.DataFrame:
     df["date"] = pd.to_datetime(df["date"])
 
@@ -439,9 +722,13 @@ with number_of_top_trafic_sites_col:
         "Number of top traffic sites", value=25
     )
 
-sla_2g_col, _ = st.columns(2)
+sla_2g_col, sla_3g_col, sla_lte_col = st.columns(3)
 with sla_2g_col:
     sla_2g = st.number_input("2G TCH availability SLA (%)", value=98.0)
+with sla_3g_col:
+    sla_3g = st.number_input("3G Cell availability SLA (%)", value=98.0)
+with sla_lte_col:
+    sla_lte = st.number_input("LTE Cell availability SLA (%)", value=98.0)
 
 if len(pre_range) != 2 or len(post_range) != 2:
     st.warning("⚠️ Please select 2 dates for each period (pre and post).")
@@ -505,6 +792,40 @@ if st.button(" Run Analysis"):
             "2G TCH availability KPI not found in input report or no data for selected periods."
         )
 
+    summary_3g_avail, site_3g_avail = analyze_3g_availability(full_df, sla_3g)
+    if summary_3g_avail is not None:
+        st.subheader("3G - Cell Availability vs SLA")
+        st.write(f"SLA target 3G Cell availability: {sla_3g}%")
+        st.dataframe(summary_3g_avail.round(2))
+
+        st.subheader("3G - Cell Availability by site (worst 25 by post-period)")
+        worst_sites_3g = site_3g_avail.sort_values("cell_avail_post").head(25)
+        st.dataframe(worst_sites_3g.round(2))
+    else:
+        st.info(
+            "3G Cell Availability KPI not found in input report or no data for selected periods."
+        )
+
+    summary_lte_avail, site_lte_avail = analyze_lte_availability(full_df, sla_lte)
+    if summary_lte_avail is not None:
+        st.subheader("LTE - Cell Availability vs SLA")
+        st.write(f"SLA target LTE Cell availability: {sla_lte}%")
+        st.dataframe(summary_lte_avail.round(2))
+
+        st.subheader("LTE - Cell Availability by site (worst 25 by post-period)")
+        worst_sites_lte = site_lte_avail.sort_values("lte_avail_post").head(25)
+        st.dataframe(worst_sites_lte.round(2))
+    else:
+        st.info(
+            "LTE Cell Availability KPI not found in input report or no data for selected periods."
+        )
+
+    # Multi-RAT availability view
+    multi_rat_df = analyze_multirat_availability(full_df, sla_2g, sla_3g, sla_lte)
+    if multi_rat_df is not None:
+        st.subheader("Multi-RAT Availability by site (post-period)")
+        st.dataframe(multi_rat_df.round(2))
+
     TraficAnalysis.last_period_df = last_period
 
 #######################################################################################################"""
@@ -674,6 +995,48 @@ if TraficAnalysis.last_period_df is not None:
 
     st.plotly_chart(fig)
 
+    # Prepare availability DataFrames for export (fallback to empty if KPI missing)
+    summary_frames = []
+    if "summary_2g_avail" in locals() and summary_2g_avail is not None:
+        tmp = summary_2g_avail.copy()
+        tmp["RAT"] = "2G"
+        summary_frames.append(tmp)
+    if "summary_3g_avail" in locals() and summary_3g_avail is not None:
+        tmp = summary_3g_avail.copy()
+        tmp["RAT"] = "3G"
+        summary_frames.append(tmp)
+    if "summary_lte_avail" in locals() and summary_lte_avail is not None:
+        tmp = summary_lte_avail.copy()
+        tmp["RAT"] = "LTE"
+        summary_frames.append(tmp)
+
+    if summary_frames:
+        availability_summary_all = pd.concat(summary_frames, ignore_index=True)
+    else:
+        availability_summary_all = pd.DataFrame()
+
+    export_site_2g = (
+        site_2g_avail
+        if "site_2g_avail" in locals() and site_2g_avail is not None
+        else pd.DataFrame()
+    )
+    export_site_3g = (
+        site_3g_avail
+        if "site_3g_avail" in locals() and site_3g_avail is not None
+        else pd.DataFrame()
+    )
+    export_site_lte = (
+        site_lte_avail
+        if "site_lte_avail" in locals() and site_lte_avail is not None
+        else pd.DataFrame()
+    )
+
+    export_multi_rat = (
+        multi_rat_df
+        if "multi_rat_df" in locals() and multi_rat_df is not None
+        else pd.DataFrame()
+    )
+
     final_dfs = convert_dfs(
         [
             full_df,
@@ -681,6 +1044,11 @@ if TraficAnalysis.last_period_df is not None:
             avg_pre_post_analysis,
             monthly_voice_df,
             monthly_data_df,
+            availability_summary_all,
+            export_site_2g,
+            export_site_3g,
+            export_site_lte,
+            export_multi_rat,
         ],
         [
             "Global_Trafic_Analysis",
@@ -688,6 +1056,11 @@ if TraficAnalysis.last_period_df is not None:
             "Avg_pre_post_analysis",
             "Monthly_voice_analysis",
             "Monthly_data_analysis",
+            "Availability_Summary_All_RAT",
+            "TwoG_Availability_By_Site",
+            "ThreeG_Availability_By_Site",
+            "LTE_Availability_By_Site",
+            "MultiRAT_Availability_By_Site",
         ],
     )
     # 📥 Bouton de téléchargement