Update app.py

app.py

@@ -1,14 +1,24 @@
 # -*- coding: utf-8 -*-
 """
 app.py — Dashboard Kekurangan Sampel IPLM (TANPA HITUNG INDEKS)
-- Fokus: melihat kekurangan jumlah sampel IPLM per wilayah
-- Bandingkan "sampel masuk (DM)" vs "populasi target (meta)"
-- Pertahankan LLM untuk membuat laporan naratif kekurangan sampel
-
-Output:
-- Tabel verifikasi (coverage & gap)
-- Download Excel (rekap + detail subset)
-- Word report (opsional pie chart kalau kaleido tersedia)
+
+Fokus:
+- Mengecek "kekurangan sampel" pengumpulan data IPLM per wilayah
+- Bandingkan sampel yang sudah masuk (DM) vs populasi target (META):
+  - Kab/Kota: SD+SMP (meta SD/SMP) dan Kec+Desa/Kel (meta jumlah desa)
+  - Provinsi: SMA (meta SMA provinsi)
+
+Fitur:
+- Filter: Provinsi, Kab/Kota, Kewenangan
+- Tabel Verifikasi Coverage & Kekurangan Sampel
+- Tabel Detail Subset DM (ringkas)
+- Download:
+  1) Rekap Excel (verifikasi + detail ringkas)
+  2) Data mentah subset DM (RAW) sesuai filter user
+  3) Laporan Word (narasi LLM + tabel verifikasi + pie ringkasan opsional)
+
+Catatan:
+- Tidak ada perhitungan Indeks IPLM sama sekali.
 """
 
 import os
@@ -26,7 +36,7 @@ from huggingface_hub import InferenceClient
 from docx import Document
 from docx.shared import Inches
 
-# Pie chart opsional (kalau kaleido ada)
+# Pie chart opsional (butuh kaleido)
 import plotly.express as px
 try:
     import kaleido  # noqa: F401
@@ -43,6 +53,7 @@ META_KAB_FILE   = "jumlahdesa_fixed (1).xlsx"   # kecamatan & desa/kel per kab/k
 META_SDSMP_FILE = "SD-SMP-kab.xlsx"             # jumlah SD & SMP per kab/kota
 META_SMA_FILE   = "SMA.xlsx"                    # jumlah SMA per provinsi
 
+
 # ============================================================
 # 1b) KONFIGURASI LLM (Hugging Face Inference)
 # ============================================================
@@ -50,7 +61,7 @@ USE_LLM = True
 LLM_MODEL_NAME = "meta-llama/Meta-Llama-3-8B-Instruct"
 
 HF_TOKEN = (
-    os.getenv("HF_SECRET")
+    os.getenv("HF_TOKEN")
     or os.getenv("HUGGINGFACEHUB_API_TOKEN")
     or os.getenv("HF_API_TOKEN")
 )
@@ -148,9 +159,27 @@ def norm_kab_label(s):
     t = " ".join(t.split())
     return re.sub(r"[^A-Z0-9]+", "", t)
 
+def safe_pct(num, den):
+    if den is None or pd.isna(den) or den <= 0:
+        return np.nan
+    if num is None or pd.isna(num):
+        num = 0
+    return 100.0 * float(num) / float(den)
+
+def _infer_jenjang_sd_smp(x):
+    if pd.isna(x):
+        return "OTHER"
+    t = str(x).upper()
+    if " SD " in f" {t} " or " SD/" in t or " MI " in f" {t} ":
+        return "SD"
+    if " SMP " in f" {t} " or " SMP/" in t or " MTS " in f" {t} ":
+        return "SMP"
+    return "OTHER"
+
 def make_pie_plotly(num, den, title):
     if not HAS_KALEIDO:
         return None
+
     if den is None or pd.isna(den) or den <= 0:
         values = [0, 1]
         labels = ["Terjangkau", "Belum Terjangkau"]
@@ -159,6 +188,7 @@ def make_pie_plotly(num, den, title):
         den = float(den)
         values = [max(num, 0), max(den - num, 0)]
         labels = ["Terjangkau", "Belum Terjangkau"]
+
     fig = px.pie(values=values, names=labels, title=title, hole=0.3)
     tmp = tempfile.mktemp(suffix=".png")
     try:
@@ -173,10 +203,16 @@ def make_pie_plotly(num, den, title):
 # ============================================================
 DATA_INFO = ""
 df_all_raw = None
-meta_kab_df = None     # kab_key -> kec, desa/kel, SD, SMP (gabungan)
-meta_sma_df = None     # prov_key -> Jml_SMA
 
-prov_col_glob = kab_col_glob = kew_col_glob = jenis_col_glob = subjenis_col_glob = nama_col_glob = None
+meta_kab_df = None   # kab_key -> (Jml_Kecamatan, Jml_DesaKel, Jml_SD, Jml_SMP)
+meta_sma_df = None   # prov_key -> Jml_SMA
+
+prov_col_glob = None
+kab_col_glob = None
+kew_col_glob = None
+jenis_col_glob = None
+subjenis_col_glob = None
+nama_col_glob = None
 
 try:
     fp = Path(DATA_FILE)
@@ -282,7 +318,6 @@ except Exception as e:
 # --- META SMA per provinsi ---
 try:
     meta_sma_raw = pd.read_excel(META_SMA_FILE)
-
     col_prov_sma = pick_col(meta_sma_raw, [
         "Provinsi", "provinsi", "PROVINSI", "NAMA_PROVINSI", "Nama Provinsi",
         "nm_prov", "nm_provinsi", "prov"
@@ -353,30 +388,7 @@ default_kew  = "KAB/KOTA" if "KAB/KOTA" in kew_choices else (kew_choices[0] if k
 # ============================================================
 # 5) INTI: HITUNG COVERAGE & GAP
 # ============================================================
-def _infer_jenjang_sd_smp(x):
-    if pd.isna(x):
-        return "OTHER"
-    t = str(x).upper()
-    # heuristik sederhana
-    if " SD " in f" {t} " or " SD/" in t or " MI " in f" {t} ":
-        return "SD"
-    if " SMP " in f" {t} " or " SMP/" in t or " MTS " in f" {t} ":
-        return "SMP"
-    return "OTHER"
-
-def safe_pct(num, den):
-    if den is None or pd.isna(den) or den <= 0:
-        return np.nan
-    if num is None or pd.isna(num):
-        num = 0
-    return 100.0 * float(num) / float(den)
-
 def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.DataFrame:
-    """
-    Keluaran: tabel coverage & GAP (kekurangan sampel) sesuai kewenangan.
-    - KAB/KOTA: bandingkan sampel sekolah vs (SD+SMP), umum vs (kec+desa/kel)
-    - PROVINSI: bandingkan sampel SMA vs (jumlah SMA)
-    """
     if df_filtered is None or len(df_filtered) == 0:
         return pd.DataFrame()
 
@@ -438,13 +450,15 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
             lambda r: safe_pct(r["Sampel_Umum"], r.get("Pop_Kec_DesaKel", np.nan)), axis=1
         )
 
-        # GAP (kekurangan sampel)
+        # GAP (kekurangan sampel) -> asumsi target = 100% populasi
         merged["Gap_Sekolah"] = merged.apply(
-            lambda r: max(int(math.ceil(r["Pop_SD_SMP"] - r["Sampel_Sekolah_Total"])) if pd.notna(r["Pop_SD_SMP"]) else 0, 0),
+            lambda r: max(int(math.ceil(r["Pop_SD_SMP"] - r["Sampel_Sekolah_Total"]))
+                          if pd.notna(r["Pop_SD_SMP"]) else 0, 0),
             axis=1
         )
         merged["Gap_Umum"] = merged.apply(
-            lambda r: max(int(math.ceil(r["Pop_Kec_DesaKel"] - r["Sampel_Umum"])) if pd.notna(r["Pop_Kec_DesaKel"]) else 0, 0),
+            lambda r: max(int(math.ceil(r["Pop_Kec_DesaKel"] - r["Sampel_Umum"]))
+                          if pd.notna(r["Pop_Kec_DesaKel"]) else 0, 0),
             axis=1
         )
 
@@ -467,7 +481,6 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
     if ("PROV" in kew_norm):
         if meta_sma_df is None:
             return pd.DataFrame({"Info": ["Meta SMA tidak tersedia."]})
-
         if prov_col_glob is None:
             return pd.DataFrame({"Info": ["Kolom provinsi tidak ditemukan di DM."]})
 
@@ -478,29 +491,32 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
 
         tmp["prov_key"] = tmp[prov_col_glob].apply(norm_prov_label)
 
+        # IMPORTANT: start dari sampel (biar tidak munculin provinsi lain dari meta)
         g_total = tmp.groupby("prov_key").size().rename("Sampel_Total").reset_index()
+
         tmp_sek = tmp[tmp["_dataset"] == "sekolah"].copy() if "_dataset" in tmp.columns else tmp.copy()
         g_sma = tmp_sek.groupby("prov_key").size().rename("Sampel_SMA").reset_index()
 
         merged = (
-            meta_sma_df.merge(g_total, on="prov_key", how="left")
-                      .merge(g_sma, on="prov_key", how="left")
+            g_total
+            .merge(g_sma, on="prov_key", how="left")
+            .merge(meta_sma_df[["prov_key", "Provinsi_Label", "Jml_SMA"]], on="prov_key", how="left")
         )
 
-        merged["Sampel_Total"] = merged["Sampel_Total"].fillna(0).astype(int)
-        merged["Sampel_SMA"]   = merged["Sampel_SMA"].fillna(0).astype(int)
+        merged["Sampel_SMA"] = merged["Sampel_SMA"].fillna(0).astype(int)
 
         merged["Coverage_SMA_%"] = merged.apply(
             lambda r: safe_pct(r["Sampel_SMA"], r.get("Jml_SMA", np.nan)), axis=1
         )
         merged["Kekurangan Sampel SMA"] = merged.apply(
-            lambda r: max(int(math.ceil(r["Jml_SMA"] - r["Sampel_SMA"])) if pd.notna(r["Jml_SMA"]) else 0, 0),
+            lambda r: max(int(math.ceil(r["Jml_SMA"] - r["Sampel_SMA"]))
+                          if pd.notna(r["Jml_SMA"]) else 0, 0),
             axis=1
         )
 
         out = pd.DataFrame({
-            "Provinsi": merged["Provinsi_Label"],
-            "Sampel Total (Prov)": merged["Sampel_Total"],
+            "Provinsi": merged["Provinsi_Label"].fillna(merged["prov_key"]),
+            "Sampel Total (Prov)": merged["Sampel_Total"].fillna(0).astype(int),
             "Sampel SMA (di DM)": merged["Sampel_SMA"],
             "Populasi SMA (Meta)": merged["Jml_SMA"],
             "Coverage SMA (%)": merged["Coverage_SMA_%"],
@@ -513,7 +529,7 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
 
 
 # ============================================================
-# 6) BUILD CONTEXT UNTUK LLM + FALLBACK
+# 6) LLM REPORT (GAP)
 # ============================================================
 def build_context_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: str) -> str:
     wilayah = kab if kab and kab != "(Semua)" else (prov if prov and prov != "(Semua)" else "NASIONAL")
@@ -522,36 +538,30 @@ def build_context_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: str) ->
     lines.append(f"Kewenangan: {kew}")
     lines.append(f"Jumlah baris verifikasi: {len(verif_df)}")
 
-    # ringkas total gap
     gap_cols = [c for c in verif_df.columns if "Kekurangan" in c]
     for gc in gap_cols:
-        try:
-            total_gap = float(pd.to_numeric(verif_df[gc], errors="coerce").fillna(0).sum())
-            lines.append(f"Total {gc}: {int(total_gap)}")
-        except Exception:
-            pass
+        total_gap = int(pd.to_numeric(verif_df[gc], errors="coerce").fillna(0).sum())
+        lines.append(f"Total {gc}: {total_gap}")
 
-    # top 10 terbesar
+    # top prioritas (ambil kolom gap pertama)
     if gap_cols:
         gc = gap_cols[0]
-        try:
-            t = verif_df.copy()
-            t[gc] = pd.to_numeric(t[gc], errors="coerce").fillna(0)
-            top = t.sort_values(gc, ascending=False).head(10)
-            keycol = "Kab/Kota" if "Kab/Kota" in top.columns else ("Provinsi" if "Provinsi" in top.columns else top.columns[0])
-            lines.append("\nTop prioritas (gap terbesar):")
-            for _, r in top.iterrows():
-                lines.append(f"- {r[keycol]}: {gc}={int(r[gc])}")
-        except Exception:
-            pass
+        t = verif_df.copy()
+        t[gc] = pd.to_numeric(t[gc], errors="coerce").fillna(0)
+        keycol = "Kab/Kota" if "Kab/Kota" in t.columns else ("Provinsi" if "Provinsi" in t.columns else t.columns[0])
+        top = t.sort_values(gc, ascending=False).head(10)
+
+        lines.append("\nTop prioritas (gap terbesar):")
+        for _, r in top.iterrows():
+            lines.append(f"- {r[keycol]}: {gc}={int(r[gc])}")
 
     return "\n".join(lines)
 
 def rule_based_gap_report(verif_df: pd.DataFrame, prov: str, kab: str, kew: str) -> str:
     if verif_df is None or verif_df.empty:
         return "Tidak ada data verifikasi yang dapat dilaporkan."
-    wilayah = kab if kab and kab != "(Semua)" else (prov if prov and prov != "(Semua)" else "NASIONAL")
 
+    wilayah = kab if kab and kab != "(Semua)" else (prov if prov and prov != "(Semua)" else "NASIONAL")
     lines = []
     lines.append("## Ringkasan Kekurangan Sampel IPLM (Rule-based)\n")
     lines.append(f"Wilayah: {wilayah}")
@@ -560,7 +570,7 @@ def rule_based_gap_report(verif_df: pd.DataFrame, prov: str, kab: str, kew: str)
 
     gap_cols = [c for c in verif_df.columns if "Kekurangan" in c]
     if not gap_cols:
-        lines.append("Kolom kekurangan sampel tidak ditemukan pada tabel verifikasi.")
+        lines.append("Kolom kekurangan sampel tidak ditemukan.")
         return "\n".join(lines)
 
     for gc in gap_cols:
@@ -568,9 +578,8 @@ def rule_based_gap_report(verif_df: pd.DataFrame, prov: str, kab: str, kew: str)
         lines.append(f"- Total {gc}: **{total_gap}** unit yang perlu dilengkapi.")
 
     lines.append(
-        "\nRekomendasi operasional: fokuskan pengumpulan data pada unit/wilayah dengan gap terbesar, "
-        "mulai dari area yang memiliki populasi target besar namun sampel masuk masih terbatas. "
-        "Pastikan konsistensi penamaan provinsi/kab-kota agar matching dengan meta tidak gagal."
+        "\nRekomendasi operasional: prioritaskan pengumpulan data pada wilayah dengan gap terbesar, "
+        "dan pastikan konsistensi penamaan provinsi/kab-kota agar pencocokan dengan meta tidak gagal."
     )
     return "\n".join(lines)
 
@@ -606,8 +615,10 @@ BATASAN:
     try:
         resp = client.chat_completion(
             model=LLM_MODEL_NAME,
-            messages=[{"role": "system", "content": system_prompt},
-                      {"role": "user", "content": user_prompt}],
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
             max_tokens=900,
             temperature=0.2,
             top_p=0.9,
@@ -629,13 +640,12 @@ BATASAN:
 # ============================================================
 def generate_word_report_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: str, analysis_text: str):
     wilayah = kab if kab and kab != "(Semua)" else (prov if prov and prov != "(Semua)" else "NASIONAL")
+
     doc = Document()
     doc.add_heading(f"Laporan Kekurangan Sampel IPLM – {wilayah}", level=1)
-
     doc.add_paragraph(f"Kewenangan: {kew}")
     doc.add_paragraph(f"Jumlah unit analisis: {len(verif_df)}")
 
-    # tabel verifikasi (batasi 200 baris biar gak jebol)
     doc.add_heading("Tabel Verifikasi Coverage & Kekurangan Sampel", level=2)
     view = verif_df.copy()
     if len(view) > 200:
@@ -652,13 +662,12 @@ def generate_word_report_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: s
         for i, c in enumerate(view.columns):
             r[i].text = str(row[c])
 
-    # pie chart opsional: hanya 1 ringkasan total (bukan per kab/prov biar gak kebanyakan)
     doc.add_heading("Ringkasan Visual (Opsional)", level=2)
     if not HAS_KALEIDO:
         doc.add_paragraph("Grafik pie tidak dibuat karena 'kaleido' tidak tersedia di server.")
     else:
-        # cari kolom pop & sampel yang paling relevan (ambil pertama yang cocok)
         pie_made = False
+        # Ringkas sekolah kab/kota
         if "Sampel Sekolah (Total)" in verif_df.columns and "Populasi Sekolah (SD+SMP)" in verif_df.columns:
             samp = pd.to_numeric(verif_df["Sampel Sekolah (Total)"], errors="coerce").fillna(0).sum()
             pop  = pd.to_numeric(verif_df["Populasi Sekolah (SD+SMP)"], errors="coerce").fillna(0).sum()
@@ -667,6 +676,7 @@ def generate_word_report_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: s
                 doc.add_picture(img, width=Inches(5))
                 pie_made = True
 
+        # Ringkas SMA provinsi
         if (not pie_made) and ("Sampel SMA (di DM)" in verif_df.columns and "Populasi SMA (Meta)" in verif_df.columns):
             samp = pd.to_numeric(verif_df["Sampel SMA (di DM)"], errors="coerce").fillna(0).sum()
             pop  = pd.to_numeric(verif_df["Populasi SMA (Meta)"], errors="coerce").fillna(0).sum()
@@ -694,7 +704,12 @@ def generate_word_report_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: s
 def run_core(prov_value, kab_value, kew_value):
     if df_all_raw is None or df_all_raw.empty:
         empty = pd.DataFrame()
-        return empty, empty, None, None, None, "Data DM tidak terbaca.", "Tidak ada analisis."
+        return (
+            empty, empty,
+            None, None, None,
+            "Data DM tidak terbaca.",
+            "Tidak ada analisis."
+        )
 
     df = df_all_raw.copy()
 
@@ -712,35 +727,52 @@ def run_core(prov_value, kab_value, kew_value):
 
     if len(df) == 0:
         empty = pd.DataFrame()
-        return empty, empty, None, None, None, "Tidak ada data untuk filter tersebut.", "Tidak ada analisis."
+        return (
+            empty, empty,
+            None, None, None,
+            "Tidak ada data untuk kombinasi filter yang dipilih.",
+            "Tidak ada analisis."
+        )
 
     # hitung verifikasi gap
     verif_df = compute_gap_verification(df, kew_value)
 
-    # buat detail subset untuk download (ringkas)
+    # detail subset untuk UI (ringkas)
     cols = []
     for c in [prov_col_glob, kab_col_glob, nama_col_glob, kew_col_glob, jenis_col_glob, subjenis_col_glob, "_dataset", "KEW_NORM"]:
         if c and c in df.columns and c not in cols:
             cols.append(c)
     detail_df = df[cols].copy() if cols else df.copy()
 
-    # simpan excel
+    # simpan file download
     tmpdir = tempfile.mkdtemp()
-    out_excel = os.path.join(tmpdir, "Kekurangan_Sampel_IPLM.xlsx")
+    rekap_excel_path = os.path.join(tmpdir, "Rekap_Kekurangan_Sampel_IPLM.xlsx")
+    raw_dm_path      = os.path.join(tmpdir, "DM_Subset_Raw.xlsx")
 
-    with pd.ExcelWriter(out_excel, engine="openpyxl") as w:
+    # 1) rekap excel (verif + detail ringkas)
+    with pd.ExcelWriter(rekap_excel_path, engine="openpyxl") as w:
         verif_df.to_excel(w, sheet_name="Verifikasi_Gap", index=False)
         detail_df.to_excel(w, sheet_name="Detail_Subset_DM", index=False)
 
-    # analisis LLM
+    # 2) raw dm subset (SEMUA kolom DM hasil filter user)
+    df.to_excel(raw_dm_path, index=False)
+
+    # 3) analisis LLM
     analysis_text = generate_llm_gap_report(verif_df, prov_value, kab_value, kew_value)
 
-    # word report
-    out_word = generate_word_report_gap(verif_df, prov_value, kab_value, kew_value, analysis_text)
+    # 4) word report
+    word_path = generate_word_report_gap(verif_df, prov_value, kab_value, kew_value, analysis_text)
 
     msg = f"OK. Subset DM: {len(df)} baris | Verifikasi: {len(verif_df)} baris."
-    return verif_df, detail_df, out_excel, out_word, None, msg, analysis_text
-
+    return (
+        verif_df,
+        detail_df,
+        rekap_excel_path,
+        raw_dm_path,
+        word_path,
+        msg,
+        analysis_text
+    )
 
 def on_prov_change(prov_value):
     return gr.update(choices=get_kab_choices_for_prov(prov_value), value="(Semua)")
@@ -787,13 +819,14 @@ Aplikasi ini hanya mengecek **kekurangan sampel** berdasarkan:
     analysis_out = gr.Markdown()
 
     with gr.Row():
-        excel_out = gr.File(label="Download Rekap Excel (.xlsx)")
-        word_out  = gr.File(label="Download Laporan Word (.docx)")
+        rekap_excel_out = gr.File(label="Download Rekap (Verifikasi + Detail) (.xlsx)")
+        raw_dm_out      = gr.File(label="Download Data Mentah Subset DM (.xlsx)")
+        word_out        = gr.File(label="Download Laporan Word (.docx)")
 
     run_btn.click(
         fn=run_core,
         inputs=[dd_prov, dd_kab, dd_kew],
-        outputs=[verif_out, detail_out, excel_out, word_out, gr.State(), msg_out, analysis_out],
+        outputs=[verif_out, detail_out, rekap_excel_out, raw_dm_out, word_out, msg_out, analysis_out],
     )
 
 demo.launch()