Add persistent availability issue detection with consecutive days tracking and criticity scoring for multi-RAT sites

apps/kpi_analysis/trafic_analysis.py

@@ -1,6 +1,6 @@
 import io
 import zipfile
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path
 
 import numpy as np
@@ -673,6 +673,159 @@ def analyze_multirat_availability(
     return multi
 
 
+def analyze_persistent_availability(
+    df: pd.DataFrame,
+    multi_rat_df: pd.DataFrame,
+    sla_2g: float,
+    sla_3g: float,
+    sla_lte: float,
+    min_consecutive_days: int = 3,
+) -> pd.DataFrame:
+    if df is None or df.empty:
+        return pd.DataFrame()
+    if "date" not in df.columns or "code" not in df.columns:
+        return pd.DataFrame()
+
+    work_df = df.copy()
+    work_df["date_only"] = work_df["date"].dt.date
+
+    site_stats = {}
+
+    def _update_stats(rat_key_prefix: str, grouped: pd.DataFrame, sla: float) -> None:
+        if grouped.empty:
+            return
+        for code, group in grouped.groupby("code"):
+            group = group.sort_values("date_only")
+            dates = pd.to_datetime(group["date_only"]).tolist()
+            below_flags = (group["value"] < sla).tolist()
+            max_streak = 0
+            current_streak = 0
+            total_below = 0
+            last_date = None
+            for flag, current_date in zip(below_flags, dates):
+                if flag:
+                    total_below += 1
+                    if (
+                        last_date is not None
+                        and current_date == last_date + timedelta(days=1)
+                        and current_streak > 0
+                    ):
+                        current_streak += 1
+                    else:
+                        current_streak = 1
+                    if current_streak > max_streak:
+                        max_streak = current_streak
+                else:
+                    current_streak = 0
+                last_date = current_date
+            stats = site_stats.setdefault(
+                code,
+                {
+                    "code": code,
+                    "max_streak_2g": 0,
+                    "max_streak_3g": 0,
+                    "max_streak_lte": 0,
+                    "below_days_2g": 0,
+                    "below_days_3g": 0,
+                    "below_days_lte": 0,
+                },
+            )
+            stats[f"max_streak_{rat_key_prefix}"] = max_streak
+            stats[f"below_days_{rat_key_prefix}"] = total_below
+
+    for rat_col, rat_key, sla in [
+        ("2g_tch_avail", "2g", sla_2g),
+        ("3g_cell_avail", "3g", sla_3g),
+        ("lte_cell_avail", "lte", sla_lte),
+    ]:
+        if rat_col in work_df.columns:
+            g = (
+                work_df.dropna(subset=[rat_col])
+                .groupby(["code", "date_only"])[rat_col]
+                .mean()
+                .reset_index()
+            )
+            g = g.rename(columns={rat_col: "value"})
+            _update_stats(rat_key, g, sla)
+
+    if not site_stats:
+        return pd.DataFrame()
+
+    rows = []
+    for code, s in site_stats.items():
+        max_2g = s.get("max_streak_2g", 0)
+        max_3g = s.get("max_streak_3g", 0)
+        max_lte = s.get("max_streak_lte", 0)
+        below_2g = s.get("below_days_2g", 0)
+        below_3g = s.get("below_days_3g", 0)
+        below_lte = s.get("below_days_lte", 0)
+        persistent_2g = max_2g >= min_consecutive_days if max_2g else False
+        persistent_3g = max_3g >= min_consecutive_days if max_3g else False
+        persistent_lte = max_lte >= min_consecutive_days if max_lte else False
+        total_below_any = below_2g + below_3g + below_lte
+        persistent_any = persistent_2g or persistent_3g or persistent_lte
+        rats_persistent_count = sum(
+            [persistent_2g is True, persistent_3g is True, persistent_lte is True]
+        )
+        rows.append(
+            {
+                "code": code,
+                "persistent_issue_2g": persistent_2g,
+                "persistent_issue_3g": persistent_3g,
+                "persistent_issue_lte": persistent_lte,
+                "max_consecutive_days_2g": max_2g,
+                "max_consecutive_days_3g": max_3g,
+                "max_consecutive_days_lte": max_lte,
+                "total_below_days_2g": below_2g,
+                "total_below_days_3g": below_3g,
+                "total_below_days_lte": below_lte,
+                "total_below_days_any": total_below_any,
+                "persistent_issue_any": persistent_any,
+                "persistent_rats_count": rats_persistent_count,
+            }
+        )
+
+    result = pd.DataFrame(rows)
+    result = result[result["persistent_issue_any"] == True]
+    if result.empty:
+        return result
+
+    if multi_rat_df is not None and not multi_rat_df.empty:
+        cols_to_merge = [
+            c
+            for c in [
+                "code",
+                "City",
+                "post_total_voice_trafic",
+                "post_total_data_trafic",
+                "post_multirat_status",
+            ]
+            if c in multi_rat_df.columns
+        ]
+        if cols_to_merge:
+            result = pd.merge(
+                result,
+                multi_rat_df[cols_to_merge].drop_duplicates("code"),
+                on="code",
+                how="left",
+            )
+
+    if "post_total_data_trafic" not in result.columns:
+        result["post_total_data_trafic"] = 0.0
+
+    result["criticity_score"] = (
+        result["post_total_data_trafic"].fillna(0) * 1.0
+        + result["total_below_days_any"].fillna(0) * 100.0
+        + result["persistent_rats_count"].fillna(0) * 1000.0
+    )
+
+    result = result.sort_values(
+        by=["criticity_score", "total_below_days_any"], ascending=[False, False]
+    )
+
+    return result
+
+
 def monthly_data_analysis(df: pd.DataFrame) -> pd.DataFrame:
     df["date"] = pd.to_datetime(df["date"])
 
@@ -771,7 +924,9 @@ if pre_range[0] < post_range[0] and pre_range[1] > post_range[1]:
     st.warning(" Pre and post periode are overlapping.")
     st.stop()
 
-if st.button(" Run Analysis"):
+run_analysis = st.button(" Run Analysis")
+
+if run_analysis:
 
     df_2g = read_uploaded_file(two_g_file)
     df_3g = read_uploaded_file(three_g_file)
@@ -794,15 +949,45 @@ if st.button(" Run Analysis"):
 
     monthly_voice_df, monthly_data_df = monthly_data_analysis(full_df)
 
+    st.session_state["full_df"] = full_df
+    st.session_state["last_period"] = last_period
+    st.session_state["sum_pre_post_analysis"] = sum_pre_post_analysis
+    st.session_state["avg_pre_post_analysis"] = avg_pre_post_analysis
+    st.session_state["monthly_voice_df"] = monthly_voice_df
+    st.session_state["monthly_data_df"] = monthly_data_df
+
+if "full_df" in st.session_state:
+    full_df = st.session_state["full_df"]
+    last_period = st.session_state["last_period"]
+    sum_pre_post_analysis = st.session_state["sum_pre_post_analysis"]
+    avg_pre_post_analysis = st.session_state["avg_pre_post_analysis"]
+    monthly_voice_df = st.session_state["monthly_voice_df"]
+    monthly_data_df = st.session_state["monthly_data_df"]
+
     full_df["week"] = full_df["date"].dt.isocalendar().week
     full_df["year"] = full_df["date"].dt.isocalendar().year
 
+    analysis_df = full_df
+    analysis_last_period = last_period
+    if "City" in full_df.columns:
+        available_cities = full_df["City"].dropna().unique()
+        if len(available_cities) > 0:
+            selected_cities = st.multiselect(
+                "Filter analysis by City (optional)",
+                sorted(available_cities),
+            )
+            if selected_cities:
+                analysis_df = full_df[full_df["City"].isin(selected_cities)].copy()
+                analysis_last_period = last_period[
+                    last_period["City"].isin(selected_cities)
+                ].copy()
+
     #  Display Summary
     st.success(" Analysis completed")
     st.subheader(" Summary Analysis Pre / Post")
     st.dataframe(sum_pre_post_analysis)
 
-    summary_2g_avail, site_2g_avail = analyze_2g_availability(full_df, sla_2g)
+    summary_2g_avail, site_2g_avail = analyze_2g_availability(analysis_df, sla_2g)
     if summary_2g_avail is not None:
         st.subheader("2G - TCH Availability vs SLA")
         st.write(f"SLA target 2G TCH availability: {sla_2g}%")
@@ -816,7 +1001,7 @@ if st.button(" Run Analysis"):
             "2G TCH availability KPI not found in input report or no data for selected periods."
         )
 
-    summary_3g_avail, site_3g_avail = analyze_3g_availability(full_df, sla_3g)
+    summary_3g_avail, site_3g_avail = analyze_3g_availability(analysis_df, sla_3g)
     if summary_3g_avail is not None:
         st.subheader("3G - Cell Availability vs SLA")
         st.write(f"SLA target 3G Cell availability: {sla_3g}%")
@@ -830,7 +1015,7 @@ if st.button(" Run Analysis"):
             "3G Cell Availability KPI not found in input report or no data for selected periods."
         )
 
-    summary_lte_avail, site_lte_avail = analyze_lte_availability(full_df, sla_lte)
+    summary_lte_avail, site_lte_avail = analyze_lte_availability(analysis_df, sla_lte)
     if summary_lte_avail is not None:
         st.subheader("LTE - Cell Availability vs SLA")
         st.write(f"SLA target LTE Cell availability: {sla_lte}%")
@@ -845,7 +1030,7 @@ if st.button(" Run Analysis"):
         )
 
     # Multi-RAT availability view
-    multi_rat_df = analyze_multirat_availability(full_df, sla_2g, sla_3g, sla_lte)
+    multi_rat_df = analyze_multirat_availability(analysis_df, sla_2g, sla_3g, sla_lte)
     if multi_rat_df is not None:
         st.subheader("Multi-RAT Availability by site (post-period)")
         st.dataframe(multi_rat_df.round(2))
@@ -924,12 +1109,132 @@ if st.button(" Run Analysis"):
                     "No LTE sites with low availability and significant traffic in post-period."
                 )
 
+        st.subheader("Persistent availability issues and critical sites")
+        min_persistent_days = st.number_input(
+            "Minimum consecutive days below SLA to flag persistent issue",
+            min_value=2,
+            max_value=30,
+            value=3,
+            step=1,
+        )
+        persistent_df = analyze_persistent_availability(
+            analysis_df, multi_rat_df, sla_2g, sla_3g, sla_lte, int(min_persistent_days)
+        )
+        if persistent_df is not None and not persistent_df.empty:
+            top_critical_n = st.number_input(
+                "Number of top critical sites to display",
+                min_value=5,
+                max_value=200,
+                value=25,
+                step=5,
+            )
+            st.dataframe(persistent_df.head(top_critical_n).round(2))
+        else:
+            st.info(
+                "No persistent availability issues detected with current parameters."
+            )
+
+    if not analysis_df.empty:
+        st.subheader("Site drill-down: traffic and availability over time")
+        sites_df = (
+            analysis_df[["code", "City"]]
+            .drop_duplicates()
+            .sort_values(by=["City", "code"])
+        )
+        site_options = sites_df.apply(
+            lambda row: (
+                f"{row['City']}_{row['code']}"
+                if pd.notna(row["City"])
+                else str(row["code"])
+            ),
+            axis=1,
+        )
+        site_map = dict(zip(site_options, sites_df["code"]))
+        selected_site_label = st.selectbox(
+            "Select a site for detailed view", options=site_options
+        )
+        selected_code = site_map.get(selected_site_label)
+        site_detail_df = analysis_df[analysis_df["code"] == selected_code].copy()
+        if not site_detail_df.empty:
+            site_detail_df = site_detail_df.sort_values("date")
+            traffic_cols = [
+                col
+                for col in ["total_voice_trafic", "total_data_trafic"]
+                if col in site_detail_df.columns
+            ]
+            if traffic_cols:
+                traffic_long = site_detail_df[["date"] + traffic_cols].melt(
+                    id_vars="date",
+                    value_vars=traffic_cols,
+                    var_name="metric",
+                    value_name="value",
+                )
+                fig_traffic = px.line(
+                    traffic_long,
+                    x="date",
+                    y="value",
+                    color="metric",
+                )
+                st.plotly_chart(fig_traffic)
+            avail_cols = []
+            rename_map = {}
+            if "2g_tch_avail" in site_detail_df.columns:
+                avail_cols.append("2g_tch_avail")
+                rename_map["2g_tch_avail"] = "2G"
+            if "3g_cell_avail" in site_detail_df.columns:
+                avail_cols.append("3g_cell_avail")
+                rename_map["3g_cell_avail"] = "3G"
+            if "lte_cell_avail" in site_detail_df.columns:
+                avail_cols.append("lte_cell_avail")
+                rename_map["lte_cell_avail"] = "LTE"
+            if avail_cols:
+                avail_df = site_detail_df[["date"] + avail_cols].copy()
+                avail_df = avail_df.rename(columns=rename_map)
+                value_cols = [c for c in avail_df.columns if c != "date"]
+                avail_long = avail_df.melt(
+                    id_vars="date",
+                    value_vars=value_cols,
+                    var_name="RAT",
+                    value_name="availability",
+                )
+                fig_avail = px.line(
+                    avail_long,
+                    x="date",
+                    y="availability",
+                    color="RAT",
+                )
+                st.plotly_chart(fig_avail)
+                site_detail_df["date_only"] = site_detail_df["date"].dt.date
+                degraded_rows_site = []
+                for rat_col, rat_name, sla_value in [
+                    ("2g_tch_avail", "2G", sla_2g),
+                    ("3g_cell_avail", "3G", sla_3g),
+                    ("lte_cell_avail", "LTE", sla_lte),
+                ]:
+                    if rat_col in site_detail_df.columns:
+                        daily_site = (
+                            site_detail_df.groupby("date_only")[rat_col].mean().dropna()
+                        )
+                        mask = daily_site < sla_value
+                        for d, val in daily_site[mask].items():
+                            degraded_rows_site.append(
+                                {
+                                    "RAT": rat_name,
+                                    "date": d,
+                                    "avg_availability": val,
+                                    "SLA": sla_value,
+                                }
+                            )
+                if degraded_rows_site:
+                    degraded_site_df = pd.DataFrame(degraded_rows_site)
+                    st.dataframe(degraded_site_df.round(2))
+
     # Temporal availability analysis - daily averages per RAT
     if any(
-        col in full_df.columns
+        col in analysis_df.columns
         for col in ["2g_tch_avail", "3g_cell_avail", "lte_cell_avail"]
     ):
-        temp_df = full_df.copy()
+        temp_df = analysis_df.copy()
         temp_df["date_only"] = temp_df["date"].dt.date
 
         agg_dict = {}
@@ -1002,7 +1307,7 @@ if st.button(" Run Analysis"):
                     st.subheader("Days with average availability below SLA")
                     st.dataframe(degraded_df.round(2))
 
-    TraficAnalysis.last_period_df = last_period
+    TraficAnalysis.last_period_df = analysis_last_period
 
 #######################################################################################################"""
 
@@ -1207,11 +1512,21 @@ if TraficAnalysis.last_period_df is not None:
         else pd.DataFrame()
     )
 
-    export_multi_rat = (
-        multi_rat_df
-        if "multi_rat_df" in locals() and multi_rat_df is not None
-        else pd.DataFrame()
+    export_multi_rat_base = analyze_multirat_availability(
+        full_df, sla_2g, sla_3g, sla_lte
     )
+    if export_multi_rat_base is not None:
+        export_multi_rat = export_multi_rat_base
+    else:
+        export_multi_rat = pd.DataFrame()
+
+    export_persistent = pd.DataFrame()
+    if export_multi_rat_base is not None:
+        export_persistent_tmp = analyze_persistent_availability(
+            full_df, export_multi_rat_base, sla_2g, sla_3g, sla_lte
+        )
+        if export_persistent_tmp is not None:
+            export_persistent = export_persistent_tmp
 
     final_dfs = convert_dfs(
         [
@@ -1225,6 +1540,7 @@ if TraficAnalysis.last_period_df is not None:
             export_site_3g,
             export_site_lte,
             export_multi_rat,
+            export_persistent,
         ],
         [
             "Global_Trafic_Analysis",
@@ -1237,6 +1553,7 @@ if TraficAnalysis.last_period_df is not None:
             "ThreeG_Availability_By_Site",
             "LTE_Availability_By_Site",
             "MultiRAT_Availability_By_Site",
+            "Top_Critical_Sites",
         ],
     )
     # 📥 Bouton de téléchargement