gsm capacity part2 with comments analysis and combinations

README.md

@@ -49,7 +49,7 @@ You can access the hosted version of the app at [https://davmelchi-db-query.hf.s
 - [x] Symetric neighbours checking
 - [x] Add the ability to select columns
 - [x] Add authentication
+- [x] Initial frequency distribution chart GSM
 - [ ] Improve Dashboard
 - [ ] Error handling
-- [ ] frequency distribution GSM
 - [ ] Add KPI analysis App

apps/kpi_analysis/gsm_capacity.py

@@ -42,6 +42,8 @@ col1, col2 = st.columns(2)
 
 threshold_col1, threshold_col2 = st.columns(2)
 threshold_col3, threshold_col4 = st.columns(2)
+max_traffic_threshold_col1, max_traffic_threshold_col2 = st.columns(2)
+
 
 if (
     uploaded_dump is not None
@@ -80,6 +82,14 @@ if (
         tch_blocking_threshold = st.number_input(
             "TCH Blocking Threshold", min_value=0.1, value=0.5
         )
+    with max_traffic_threshold_col1:
+        max_traffic_threshold = st.number_input(
+            "TCH Utilization Max Traffic Threshold", min_value=0, value=90
+        )
+    # with max_traffic_threshold_col2:
+    #     max_traffic_threshold = st.number_input(
+    #         "Max Traffic Threshold", min_value=0, value=10
+    #     )
 
     if st.button("Analyze Data", type="primary"):
         dfs = analyze_gsm_data(
@@ -92,12 +102,13 @@ if (
             tch_abis_fails_threshold=tch_abis_fails_threshold,
             sdcch_blocking_threshold=sdcch_blocking_threshold,
             tch_blocking_threshold=tch_blocking_threshold,
+            max_traffic_threshold=max_traffic_threshold,
         )
 
         if dfs is not None:
-            gsm_analysis_df = dfs[0]
-            bh_kpi_df = dfs[1]
-            daily_kpi_df = dfs[2]
+            gsm_analysis_df: pd.DataFrame = dfs[0]
+            bh_kpi_df: pd.DataFrame = dfs[1]
+            daily_kpi_df: pd.DataFrame = dfs[2]
             GsmCapacity.final_results = convert_gsm_dfs(
                 [gsm_analysis_df, bh_kpi_df, daily_kpi_df],
                 ["GSM_Analysis", "BH_KPI_Analysis", "Daily_KPI_Analysis"],
@@ -118,3 +129,36 @@ if (
                 )
 
         st.write(daily_kpi_df)
+
+        # Add dataframe and ploty bar chart with "Final comment" distribution in gsm_analysis_df in 2 columns
+        final_comments_df = (
+            gsm_analysis_df.groupby("Final comment").size().reset_index(name="count")
+        )
+        fig = px.bar(final_comments_df, x="Final comment", y="count")
+        fig.update_layout(height=1500)
+        fig.update_traces(texttemplate="%{value}", textposition="outside")
+        st.plotly_chart(fig, use_container_width=True)
+        st.write(final_comments_df)
+
+        # Add dataframe and ploty bar chart with "BH Congestion status" distribution in gsm_analysis_df in 2 columns
+        bh_congestion_status_df = (
+            gsm_analysis_df.groupby("BH Congestion status")
+            .size()
+            .reset_index(name="count")
+        )
+        fig = px.bar(bh_congestion_status_df, x="BH Congestion status", y="count")
+        fig.update_layout(height=1500)
+        fig.update_traces(texttemplate="%{value}", textposition="outside")
+        st.plotly_chart(fig, use_container_width=True)
+        st.write(bh_congestion_status_df)
+        # Add dataframe and ploty bar chart with "operational_comment" distribution in gsm_analysis_df in 2 columns
+        operational_comments_df = (
+            gsm_analysis_df.groupby("operational_comment")
+            .size()
+            .reset_index(name="count")
+        )
+        fig = px.bar(operational_comments_df, x="operational_comment", y="count")
+        fig.update_layout(height=1500)
+        fig.update_traces(texttemplate="%{value}", textposition="outside")
+        st.plotly_chart(fig, use_container_width=True)
+        st.write(operational_comments_df)

process_kpi/process_gsm_capacity.py

@@ -6,6 +6,9 @@ from utils.check_sheet_exist import execute_checks_sheets_exist
 from utils.convert_to_excel import convert_dfs, save_dataframe
 from utils.kpi_analysis_utils import (
     GsmAnalysis,
+    analyze_sdcch_call_blocking,
+    analyze_tch_abis_fails,
+    analyze_tch_call_blocking,
     cell_availability_analysis,
     combine_comments,
     create_daily_date,
@@ -74,114 +77,19 @@ BH_COLUMNS_FOR_CAPACITY = [
     "max_sdcch_real_blocking_bh",
     "avg_sdcch_real_blocking_bh",
     "number_of_days_with_sdcch_blocking_exceeded_bh",
+    "tch_call_blocking_bh_comment",
+    "sdcch_real_blocking_bh_comment",
 ]
 
-
-def analyze_tch_abis_fails(
-    df: pd.DataFrame,
-    number_of_kpi_days: int,
-    analysis_type: str,
-    number_of_threshold_days: int,
-    tch_abis_fails_threshold: int,
-) -> pd.DataFrame:
-
-    result_df = df.copy()
-    last_days_df = result_df.iloc[:, -number_of_kpi_days:]
-    # last_days_df = last_days_df.fillna(0)
-
-    result_df[f"avg_tch_abis_fail_{analysis_type.lower()}"] = last_days_df.mean(
-        axis=1
-    ).round(2)
-    result_df[f"max_tch_abis_fail_{analysis_type.lower()}"] = last_days_df.max(axis=1)
-    # Count the number of days above threshold
-    result_df[f"number_of_days_with_tch_abis_fail_exceeded_{analysis_type.lower()}"] = (
-        last_days_df.apply(
-            lambda row: sum(1 for x in row if x >= tch_abis_fails_threshold), axis=1
-        )
-    )
-
-    # Add the daily_tch_comment : if number_of_days_with_tch_abis_fail_exceeded_daily is >= number_of_threshold_days : tch abis fail exceeded treshold , else : None
-    result_df[f"tch_abis_fail_{analysis_type.lower()}_comment"] = np.where(
-        result_df[f"number_of_days_with_tch_abis_fail_exceeded_{analysis_type.lower()}"]
-        >= number_of_threshold_days,
-        "tch abis fail exceeded treshold",
-        None,
-    )
-
-    return result_df
-
-
-def analyze_tch_call_blocking(
-    df: pd.DataFrame,
-    number_of_kpi_days: int,
-    analysis_type: str,
-    number_of_threshold_days: int,
-    tch_blocking_threshold: int,
-) -> pd.DataFrame:
-
-    result_df = df.copy()
-    last_days_df = result_df.iloc[:, -number_of_kpi_days:]
-    # last_days_df = last_days_df.fillna(0)
-
-    result_df[f"avg_tch_call_blocking_{analysis_type.lower()}"] = last_days_df.mean(
-        axis=1
-    ).round(2)
-    result_df[f"max_tch_call_blocking_{analysis_type.lower()}"] = last_days_df.max(
-        axis=1
-    )
-    # Count the number of days above threshold
-    result_df[f"number_of_days_with_tch_blocking_exceeded_{analysis_type.lower()}"] = (
-        last_days_df.apply(
-            lambda row: sum(1 for x in row if x >= tch_blocking_threshold), axis=1
-        )
-    )
-
-    # Add the daily_tch_comment : if number_of_days_with_tch_blocking_exceeded_daily is >= number_of_threshold_days : tch blocking exceeded treshold , else : None
-    result_df[f"tch_call_blocking_{analysis_type.lower()}_comment"] = np.where(
-        result_df[f"number_of_days_with_tch_blocking_exceeded_{analysis_type.lower()}"]
-        >= number_of_threshold_days,
-        "TCH blocking exceeded threshold",
-        None,
-    )
-    return result_df
-
-
-def analyze_sdcch_call_blocking(
-    df: pd.DataFrame,
-    number_of_kpi_days: int,
-    sdcch_blocking_threshold: int,
-    analysis_type: str,
-    number_of_threshold_days: int,
-) -> pd.DataFrame:
-
-    result_df = df.copy()
-    last_days_df = result_df.iloc[:, -number_of_kpi_days:]
-    # last_days_df = last_days_df.fillna(0)
-
-    result_df[f"avg_sdcch_real_blocking_{analysis_type.lower()}"] = last_days_df.mean(
-        axis=1
-    ).round(2)
-    result_df[f"max_sdcch_real_blocking_{analysis_type.lower()}"] = last_days_df.max(
-        axis=1
-    )
-    # Count the number of days above threshold
-    result_df[
-        f"number_of_days_with_sdcch_blocking_exceeded_{analysis_type.lower()}"
-    ] = last_days_df.apply(
-        lambda row: sum(1 for x in row if x >= sdcch_blocking_threshold), axis=1
-    )
-
-    # add daily_sdcch_comment : if number_of_days_with_sdcch_blocking_exceeded_daily is >= number_of_threshold_days : sdcch blocking exceeded treshold , else : None
-    result_df[f"sdcch_real_blocking_{analysis_type.lower()}_comment"] = np.where(
-        result_df[
-            f"number_of_days_with_sdcch_blocking_exceeded_{analysis_type.lower()}"
-        ]
-        >= number_of_threshold_days,
-        "SDCCH blocking exceeded threshold",
-        None,
-    )
-
-    return result_df
+DAILY_COLUMNS_FOR_CAPACITY = [
+    "Average_cell_availability",
+    "number_of_days_exceeding_threshold",
+    "availability_comment",
+    "avg_tch_abis_fail_daily",
+    "max_tch_abis_fail_daily",
+    "number_of_days_with_tch_abis_fail_exceeded_daily",
+    "tch_abis_fail_daily_comment",
+]
 
 
 def bh_traffic_analysis(
@@ -190,7 +98,7 @@ def bh_traffic_analysis(
 ) -> pd.DataFrame:
 
     result_df = df.copy()
-    last_days_df = result_df.iloc[:, -number_of_kpi_days:]
+    last_days_df: pd.DataFrame = result_df.iloc[:, -number_of_kpi_days:]
     # last_days_df = last_days_df.fillna(0)
 
     result_df["Avg_Traffic BH"] = last_days_df.mean(axis=1).round(2)
@@ -282,6 +190,7 @@ def analyse_bh_data(
         number_of_kpi_days=number_of_kpi_days,
         tch_blocking_threshold=tch_blocking_threshold,
         sdcch_blocking_threshold=sdcch_blocking_threshold,
+        number_of_threshold_days=number_of_threshold_days,
     )
 
     bh_df_for_capacity = df.copy()
@@ -416,7 +325,19 @@ def analyse_daily_data(
         sdcch_blocking_threshold=sdcch_blocking_threshold,
         tch_blocking_threshold=tch_blocking_threshold,
     )
-    return df
+    daily_df_for_capacity = df.copy()
+    daily_df_for_capacity = daily_df_for_capacity[DAILY_COLUMNS_FOR_CAPACITY]
+    daily_df_for_capacity = daily_df_for_capacity.reset_index()
+
+    if isinstance(daily_df_for_capacity.columns, pd.MultiIndex):
+        daily_df_for_capacity.columns = [
+            "_".join([str(el) for el in col if el])
+            for col in daily_df_for_capacity.columns.values
+        ]
+    # Rename "BTS_name" to "name"
+    daily_df_for_capacity = daily_df_for_capacity.rename(columns={"BTS_name": "name"})
+
+    return daily_df_for_capacity, df
 
 
 def get_gsm_databases(dump_path: str) -> pd.DataFrame:
@@ -468,9 +389,10 @@ def analyze_gsm_data(
     tch_abis_fails_threshold: int,
     sdcch_blocking_threshold: float,
     tch_blocking_threshold: float,
+    max_traffic_threshold: int,
 ):
 
-    daily_kpi_df: pd.DataFrame = analyse_daily_data(
+    daily_kpi_dfs: pd.DataFrame = analyse_daily_data(
         daily_report_path=daily_report_path,
         number_of_kpi_days=number_of_kpi_days,
         availability_threshold=availability_threshold,
@@ -493,13 +415,32 @@ def analyze_gsm_data(
     bh_kpi_df = bh_kpi_dfs[0]
     bh_kpi_full_df = bh_kpi_dfs[1]
 
+    daily_kpi_df = daily_kpi_dfs[0]
+    daily_kpi_full_df = daily_kpi_dfs[1]
+
     gsm_analysis_df = gsm_database_df.merge(bh_kpi_df, on="name", how="left")
+    gsm_analysis_df = gsm_analysis_df.merge(daily_kpi_df, on="name", how="left")
 
     # "TCH UTILIZATION (@Max Traffic)" equal to "(Max_Trafic" divided by "Offered Traffic BH)*100"
     gsm_analysis_df["TCH UTILIZATION (@Max Traffic)"] = (
         gsm_analysis_df["Max_Traffic BH"] / gsm_analysis_df["Offered Traffic BH"]
     ) * 100
 
+    # Add column "Tch utilization comments" : if "TCH UTILIZATION (@Max Traffic)" exceeded it's threshold then "Tch utilization exceeded threshold else None
+    gsm_analysis_df["Tch utilization comments"] = np.where(
+        gsm_analysis_df["TCH UTILIZATION (@Max Traffic)"] > max_traffic_threshold,
+        "Tch utilization exceeded threshold",
+        None,
+    )
+    # Add "BH Congestion status" : concatenate "Tch utilization comments" + "tch_call_blocking_bh_comment" + "sdcch_real_blocking_bh_comment"
+    gsm_analysis_df = combine_comments(
+        gsm_analysis_df,
+        "Tch utilization comments",
+        "tch_call_blocking_bh_comment",
+        "sdcch_real_blocking_bh_comment",
+        new_column="BH Congestion status",
+    )
+
     # Add "ERLANGB value" =MAX TRAFFIC/(1-(MAX TCH call blocking/200))
     gsm_analysis_df["ErlabngB_value"] = gsm_analysis_df["Max_Traffic BH"] / (
         1 - (gsm_analysis_df["max_tch_call_blocking_bh"] / 200)
@@ -530,4 +471,47 @@ def analyze_gsm_data(
         gsm_analysis_df["Target TRXs"] - gsm_analysis_df["number_trx_per_cell"]
     )
 
-    return [gsm_analysis_df, bh_kpi_full_df, daily_kpi_df]
+    # if "availability_comment" equal to "Down Site" then "Down Site"
+    # if "availability_comment" is not "Availability OK" and  "tch_abis_fail_daily_comment" equal to "tch abis fail exceeded threshold" then "Availability and TX issues"
+    # if "availability_comment" is not "Availability OK" and  "tch_abis_fail_daily_comment" is empty then "Availability issues"
+    # if "availability_comment" is "Availability OK" and  "tch_abis_fail_daily_comment" equal to "tch abis fail exceeded threshold" then "TX issues"
+    # Else "Operational is OK"
+    gsm_analysis_df["operational_comment"] = np.select(
+        [
+            gsm_analysis_df["availability_comment"] == "Down Site",  # 1
+            (gsm_analysis_df["availability_comment"] != "Availability OK")
+            & (
+                gsm_analysis_df["tch_abis_fail_daily_comment"]
+                == "tch abis fail exceeded threshold"
+            ),  # 2
+            (gsm_analysis_df["availability_comment"] != "Availability OK")
+            & pd.isna(gsm_analysis_df["tch_abis_fail_daily_comment"]),  # 3
+            (gsm_analysis_df["availability_comment"] == "Availability OK")
+            & (
+                gsm_analysis_df["tch_abis_fail_daily_comment"]
+                == "tch abis fail exceeded threshold"
+            ),  # 4
+        ],
+        [
+            "Down Site",  # 1
+            "Availability and TX issues",  # 2
+            "Availability issues",  # 3
+            "TX issues",  # 4
+        ],
+        default="Operational is OK",
+    )
+
+    # Add "Final comment" with "BH Congestion status" + "operational_comment"
+    gsm_analysis_df = combine_comments(
+        gsm_analysis_df,
+        "BH Congestion status",
+        "operational_comment",
+        new_column="Final comment",
+    )
+
+    # Replace 'nan, nan, Availability issues' with None
+    # gsm_analysis_df["Final comment"] = gsm_analysis_df["Final comment"].replace(
+    #     "nan, nan, Availability issues", None
+    # )
+
+    return [gsm_analysis_df, bh_kpi_full_df, daily_kpi_full_df]

utils/kpi_analysis_utils.py

@@ -1,3 +1,4 @@
+import numpy as np
 import pandas as pd
 
 
@@ -230,7 +231,7 @@ def combine_comments(df: pd.DataFrame, *columns: str, new_column: str) -> pd.Dat
     """
     result_df = df.copy()
     result_df[new_column] = result_df[list(columns)].apply(
-        lambda row: ", ".join([x for x in row if x]), axis=1
+        lambda row: ", ".join([str(x) for x in row if x]), axis=1
     )
     # Trim all trailing commas
     result_df[new_column] = result_df[new_column].str.replace(
@@ -384,3 +385,110 @@ def cell_availability_analysis(
     )
 
     return result_df
+
+
+def analyze_tch_abis_fails(
+    df: pd.DataFrame,
+    number_of_kpi_days: int,
+    analysis_type: str,
+    number_of_threshold_days: int,
+    tch_abis_fails_threshold: int,
+) -> pd.DataFrame:
+
+    result_df = df.copy()
+    last_days_df: pd.DataFrame = result_df.iloc[:, -number_of_kpi_days:]
+    # last_days_df = last_days_df.fillna(0)
+
+    result_df[f"avg_tch_abis_fail_{analysis_type.lower()}"] = last_days_df.mean(
+        axis=1
+    ).round(2)
+    result_df[f"max_tch_abis_fail_{analysis_type.lower()}"] = last_days_df.max(axis=1)
+    # Count the number of days above threshold
+    result_df[f"number_of_days_with_tch_abis_fail_exceeded_{analysis_type.lower()}"] = (
+        last_days_df.apply(
+            lambda row: sum(1 for x in row if x >= tch_abis_fails_threshold), axis=1
+        )
+    )
+
+    # Add the daily_tch_comment : if number_of_days_with_tch_abis_fail_exceeded_daily is >= number_of_threshold_days : tch abis fail exceeded threshold , else : None
+    result_df[f"tch_abis_fail_{analysis_type.lower()}_comment"] = np.where(
+        result_df[f"number_of_days_with_tch_abis_fail_exceeded_{analysis_type.lower()}"]
+        >= number_of_threshold_days,
+        "tch abis fail exceeded threshold",
+        None,
+    )
+
+    return result_df
+
+
+def analyze_tch_call_blocking(
+    df: pd.DataFrame,
+    number_of_kpi_days: int,
+    analysis_type: str,
+    number_of_threshold_days: int,
+    tch_blocking_threshold: int,
+) -> pd.DataFrame:
+
+    result_df = df.copy()
+    last_days_df: pd.DataFrame = result_df.iloc[:, -number_of_kpi_days:]
+    # last_days_df = last_days_df.fillna(0)
+
+    result_df[f"avg_tch_call_blocking_{analysis_type.lower()}"] = last_days_df.mean(
+        axis=1
+    ).round(2)
+    result_df[f"max_tch_call_blocking_{analysis_type.lower()}"] = last_days_df.max(
+        axis=1
+    )
+    # Count the number of days above threshold
+    result_df[f"number_of_days_with_tch_blocking_exceeded_{analysis_type.lower()}"] = (
+        last_days_df.apply(
+            lambda row: sum(1 for x in row if x >= tch_blocking_threshold), axis=1
+        )
+    )
+
+    # Add the daily_tch_comment : if number_of_days_with_tch_blocking_exceeded_daily is >= number_of_threshold_days : tch blocking exceeded threshold , else : None
+    result_df[f"tch_call_blocking_{analysis_type.lower()}_comment"] = np.where(
+        result_df[f"number_of_days_with_tch_blocking_exceeded_{analysis_type.lower()}"]
+        >= number_of_threshold_days,
+        "TCH blocking exceeded threshold",
+        None,
+    )
+    return result_df
+
+
+def analyze_sdcch_call_blocking(
+    df: pd.DataFrame,
+    number_of_kpi_days: int,
+    sdcch_blocking_threshold: int,
+    analysis_type: str,
+    number_of_threshold_days: int,
+) -> pd.DataFrame:
+
+    result_df = df.copy()
+    last_days_df: pd.DataFrame = result_df.iloc[:, -number_of_kpi_days:]
+    # last_days_df = last_days_df.fillna(0)
+
+    result_df[f"avg_sdcch_real_blocking_{analysis_type.lower()}"] = last_days_df.mean(
+        axis=1
+    ).round(2)
+    result_df[f"max_sdcch_real_blocking_{analysis_type.lower()}"] = last_days_df.max(
+        axis=1
+    )
+    # Count the number of days above threshold
+    result_df[
+        f"number_of_days_with_sdcch_blocking_exceeded_{analysis_type.lower()}"
+    ] = last_days_df.apply(
+        lambda row: sum(1 for x in row if x >= sdcch_blocking_threshold), axis=1
+    )
+
+    # add daily_sdcch_comment : if number_of_days_with_sdcch_blocking_exceeded_daily is >= number_of_threshold_days : sdcch blocking exceeded threshold , else : None
+    result_df[f"sdcch_real_blocking_{analysis_type.lower()}_comment"] = np.where(
+        result_df[
+            f"number_of_days_with_sdcch_blocking_exceeded_{analysis_type.lower()}"
+        ]
+        >= number_of_threshold_days,
+        "SDCCH blocking exceeded threshold",
+        None,
+    )
+
+    return result_df