Update app.py

app.py

@@ -40,12 +40,12 @@ except Exception:
 
 
 # ============================================================
-# 1) KONFIGURASI FILE
+# 1) KONFIGURASI FILE (DISESUAIKAN DENGAN FILE UPLOAD KAMU)
 # ============================================================
-DATA_FILE       = "IPLM_clean_Manual.xlsx"                 # DM sampel masuk (multi-sheet)
-META_KAB_FILE   = "jumlahdesa_fixed (1).xlsx"   # kecamatan & desa/kel per kab/kota
-META_SDSMP_FILE = "SD-SMP-kab.xlsx"             # jumlah SD & SMP per kab/kota
-META_SMA_FILE   = "SMA.xlsx"                    # jumlah SMA per provinsi
+DATA_FILE       = "IPLM_clean_manual_131225.xlsx"      # DM sampel masuk (multi-sheet)
+META_KAB_FILE   = "Data_populasi_Kab_kota.xlsx"        # populasi kab/kota (kec, desa, SD, SMP)
+META_SDSMP_FILE = "Data_populasi_Kab_kota.xlsx"        # tetap ada (logic sama), sumber sama
+META_SMA_FILE   = "Data_populasi_propinsi.xlsx"        # populasi provinsi (SMA)
 
 # ============================================================
 # 1a) TARGET CAKUPAN (KEBIJAKAN)
@@ -163,10 +163,12 @@ def clean_prov_display(s):
         return None
     t = str(s).upper().strip()
     t = " ".join(t.split())
-    # hilangkan prefix PROVINSI berulang
     while t.startswith("PROVINSI PROVINSI "):
         t = t.replace("PROVINSI PROVINSI ", "PROVINSI ", 1)
     t = t.replace("PROVINSI PROVINSI ", "PROVINSI ")
+    if not t.startswith("PROVINSI "):
+        # di DM kamu kolom 'provinsi' biasanya tanpa prefix PROVINSI, biar konsisten kita tambahin
+        t = "PROVINSI " + t
     return t
 
 def clean_kab_display(s):
@@ -174,10 +176,8 @@ def clean_kab_display(s):
         return None
     t = str(s).upper().strip()
     t = " ".join(t.split())
-    # rapihin kab/kota
     t = t.replace("KABUPATEN", "KAB.")
     t = t.replace("KAB  ", "KAB. ")
-    t = t.replace("KAB.", "KAB.")
     t = t.replace("KOTA ADMINISTRASI", "KOTA")
     return t
 
@@ -207,7 +207,7 @@ def make_pie_plotly(num, den, title):
 DATA_INFO = ""
 df_all_raw = None
 
-meta_kab_df = None   # kab_key -> (Jml_Kecamatan, Jml_DesaKel, Jml_SD, Jml_SMP)
+meta_kab_df = None   # kab_key -> (Jml_Kecamatan, Jml_DesaKel, Jml_SD, Jml_SMP, optional populasi_umum/pop_sekolah)
 meta_sma_df = None   # prov_key -> (Jml_SMA)
 
 prov_col_glob = None
@@ -229,12 +229,13 @@ try:
     frames = [pd.read_excel(fp, sheet_name=s) for s in xls.sheet_names]
     df_all_raw = pd.concat(frames, ignore_index=True, sort=False)
 
+    # >>> DISESUAIKAN: DM kamu punya kolom 'provinsi', 'kab_kota', dst.
     prov_col_glob  = pick_col(df_all_raw, ["provinsi", "Provinsi", "PROVINSI"])
-    kab_col_glob   = pick_col(df_all_raw, ["kab_kota", "Kab_Kota", "Kab/Kota", "KAB/KOTA", "kabupaten_kota", "kota"])
+    kab_col_glob   = pick_col(df_all_raw, ["kab_kota", "kab/kota", "Kab/Kota", "KAB/KOTA", "kabupaten_kota", "kota"])
     kew_col_glob   = pick_col(df_all_raw, ["kewenangan", "jenis_kewenangan", "Kewenangan", "KEWENANGAN"])
-    jenis_col_glob = pick_col(df_all_raw, ["jenis_perpustakaan", "JENIS_PERPUSTAKAAN", "Jenis Perpustakaan", "jenis perpustakaan"])
+    jenis_col_glob = pick_col(df_all_raw, ["jenis_perpustakaan", "JENIS_PERPUSTAKAAN", "Jenis Perpustakaan"])
     subjenis_col_glob = pick_col(df_all_raw, ["sub_jenis_perpus", "Sub Jenis", "SubJenis", "subjenis", "jenjang"])
-    nama_col_glob  = pick_col(df_all_raw, ["nama_perpustakaan", "nm_perpustakaan", "nm_instansi_lembaga", "Nama Perpustakaan"])
+    nama_col_glob  = pick_col(df_all_raw, ["nm_perpustakaan", "nama_perpustakaan", "nm_instansi_lembaga", "Nama Perpustakaan"])
 
     # kewenangan normal
     if kew_col_glob:
@@ -251,6 +252,8 @@ try:
         "PERPUSTAKAAN DAERAH": "umum",
         "PERPUSTAKAAN KHUSUS": "khusus",
         "KHUSUS": "khusus",
+        "PERPUSTAKAAN PERGURUAN TINGGI": "khusus",
+        "PERGURUAN TINGGI": "khusus",
     }
     if jenis_col_glob:
         df_all_raw["_dataset"] = df_all_raw[jenis_col_glob].apply(_norm_text).map(val_map_jenis)
@@ -273,33 +276,65 @@ except Exception as e:
     df_all_raw = None
     DATA_INFO = f"⚠️ Gagal memuat `{DATA_FILE}` | Error: `{e}`"
 
-# ---- Meta Kab (Kec/Desa) ----
+# ---- Meta Kab/Kota (Populasi) ----
 try:
     meta_kab_raw = pd.read_excel(META_KAB_FILE)
-    col_kab = pick_col(meta_kab_raw, ["Kab/Kota", "Kab_Kota", "kab/kota", "kabupaten_kota"])
-    col_kec = pick_col(meta_kab_raw, ["Kecamatan", "jml_kecamatan", "jumlah_kecamatan"])
-    col_des = pick_col(meta_kab_raw, ["Desa/Kel", "Desa Kelurahan", "Desa", "Desa_kel"])
 
-    if col_kab and col_kec and col_des:
+    # >>> DISESUAIKAN: kolom file kamu:
+    # 'KABUPATEN_KOTA', 'Jml Kec', 'Jml Desa', 'SD', 'SMP', 'jumlah_populasi_umum', 'jumlah_populasi_sekolah'
+    col_kab = pick_col(meta_kab_raw, ["KABUPATEN_KOTA", "KAB/KOTA", "Kab/Kota", "Kab_Kota", "kab/kota", "kabupaten_kota"])
+    col_kec = pick_col(meta_kab_raw, ["Jml Kec", "JML KEC", "Jml Kecamatan", "Kecamatan", "jml_kecamatan", "jumlah_kecamatan"])
+    col_des = pick_col(meta_kab_raw, ["Jml Desa", "JML DESA", "Jml Desa/Kel", "Desa/Kel", "Desa Kelurahan", "Desa", "Desa_kel"])
+
+    col_sd  = pick_col(meta_kab_raw, ["SD", "Jumlah SD", "Total SD", "SD_Total", "jml_sd", "Jml_SD"])
+    col_smp = pick_col(meta_kab_raw, ["SMP", "Jumlah SMP", "Total SMP", "SMP_Total", "jml_smp", "Jml_SMP"])
+
+    col_pop_umum    = pick_col(meta_kab_raw, ["jumlah_populasi_umum", "Populasi Umum", "pop_umum"])
+    col_pop_sekolah = pick_col(meta_kab_raw, ["jumlah_populasi_sekolah", "Populasi Sekolah", "pop_sekolah"])
+
+    if col_kab:
         meta_kab_df = pd.DataFrame({
             "Kab_Kota_Label": meta_kab_raw[col_kab].astype(str).str.strip(),
-            "Jml_Kecamatan":  meta_kab_raw[col_kec].apply(coerce_num),
-            "Jml_DesaKel":    meta_kab_raw[col_des].apply(coerce_num),
         })
+        meta_kab_df["Jml_Kecamatan"] = meta_kab_raw[col_kec].apply(coerce_num) if col_kec else np.nan
+        meta_kab_df["Jml_DesaKel"]   = meta_kab_raw[col_des].apply(coerce_num) if col_des else np.nan
+        meta_kab_df["Jml_SD"]        = meta_kab_raw[col_sd].apply(coerce_num) if col_sd else np.nan
+        meta_kab_df["Jml_SMP"]       = meta_kab_raw[col_smp].apply(coerce_num) if col_smp else np.nan
+
+        if col_pop_umum:
+            meta_kab_df["Pop_Umum_Meta"] = meta_kab_raw[col_pop_umum].apply(coerce_num)
+        else:
+            meta_kab_df["Pop_Umum_Meta"] = np.nan
+
+        if col_pop_sekolah:
+            meta_kab_df["Pop_Sekolah_Meta"] = meta_kab_raw[col_pop_sekolah].apply(coerce_num)
+        else:
+            meta_kab_df["Pop_Sekolah_Meta"] = np.nan
+
         meta_kab_df["kab_key"] = meta_kab_df["Kab_Kota_Label"].apply(norm_kab_label)
-        extra_info.append(f"Meta Kab/Kota (Kec/Desa) terbaca: **{META_KAB_FILE}** (n={len(meta_kab_df)})")
+        meta_kab_df = meta_kab_df.groupby("kab_key", as_index=False).agg({
+            "Kab_Kota_Label": "first",
+            "Jml_Kecamatan": "first",
+            "Jml_DesaKel": "first",
+            "Jml_SD": "first",
+            "Jml_SMP": "first",
+            "Pop_Umum_Meta": "first",
+            "Pop_Sekolah_Meta": "first",
+        })
+
+        extra_info.append(f"Meta Kab/Kota terbaca: **{META_KAB_FILE}** (n={len(meta_kab_df)})")
     else:
         meta_kab_df = None
-        extra_info.append(f"⚠️ Kolom kunci meta kab tidak lengkap di `{META_KAB_FILE}`")
+        extra_info.append(f"⚠️ Kolom kab/kota tidak ditemukan di `{META_KAB_FILE}`")
 except Exception as e:
     meta_kab_df = None
     extra_info.append(f"⚠️ Gagal memuat `{META_KAB_FILE}` ({e})")
 
-# ---- Meta SD/SMP ----
+# ---- Meta SD/SMP (tetap dipertahankan logic-nya, sumber sama) ----
 try:
     sd_smp_raw = pd.read_excel(META_SDSMP_FILE)
     col_kab2 = pick_col(sd_smp_raw, [
-        "Kabupaten/Kota_Kabupaten/Kota", "Kabupaten/Kota",
+        "KABUPATEN_KOTA", "Kabupaten/Kota_Kabupaten/Kota", "Kabupaten/Kota",
         "Kab/Kota", "Kab_Kota", "kab/kota", "kabupaten_kota"
     ])
     col_sd   = pick_col(sd_smp_raw, ["SD", "Jumlah SD", "Total SD", "SD_Total", "jml_sd", "Jml_SD"])
@@ -319,7 +354,13 @@ try:
         })
 
         if meta_kab_df is not None:
-            meta_kab_df = meta_kab_df.merge(df_sd_smp_grp, on="kab_key", how="left")
+            meta_kab_df = meta_kab_df.merge(df_sd_smp_grp, on="kab_key", how="left", suffixes=("", "_x"))
+            # kalau dari meta_kab_df SD/SMP kosong, isi dari grup
+            meta_kab_df["Jml_SD"]  = meta_kab_df["Jml_SD"].fillna(meta_kab_df.get("Jml_SD_x"))
+            meta_kab_df["Jml_SMP"] = meta_kab_df["Jml_SMP"].fillna(meta_kab_df.get("Jml_SMP_x"))
+            for c in ["Jml_SD_x", "Jml_SMP_x"]:
+                if c in meta_kab_df.columns:
+                    meta_kab_df = meta_kab_df.drop(columns=[c])
         else:
             meta_kab_df = df_sd_smp_grp.copy()
             meta_kab_df["Kab_Kota_Label"] = df_sd_smp.groupby("kab_key")["Kab_Kota_Label_SD"].first().values
@@ -330,22 +371,28 @@ try:
 except Exception as e:
     extra_info.append(f"⚠️ Gagal memuat `{META_SDSMP_FILE}` ({e})")
 
-# ---- Meta SMA ----
+# ---- Meta SMA (Provinsi) ----
 try:
     meta_sma_raw = pd.read_excel(META_SMA_FILE)
+
+    # >>> DISESUAIKAN: kolom file kamu:
+    # 'Provinsi', 'sma ' (ada spasi), dan beberapa kolom lain
     col_prov_sma = pick_col(meta_sma_raw, [
         "Provinsi", "provinsi", "PROVINSI", "NAMA_PROVINSI", "Nama Provinsi",
         "nm_prov", "nm_provinsi", "prov"
     ])
+
+    # prioritas "sma " (dengan spasi), lalu variasi lain
     col_sma = pick_col(meta_sma_raw, [
-        "Total SMA", "TOTAL_SMA", "TOTAL", "total",
-        "Jml_SMA", "Jumlah SMA", "SMA", "SMA_Total",
+        "sma ", "SMA", "Total SMA", "TOTAL_SMA", "TOTAL", "total",
+        "Jml_SMA", "Jumlah SMA", "SMA_Total",
         "jumlah_sma", "total_sma", "jml_sma"
     ])
+
     if col_prov_sma is None:
-        raise ValueError("Kolom provinsi tidak ditemukan di file SMA.")
+        raise ValueError("Kolom provinsi tidak ditemukan di file populasi provinsi.")
     if col_sma is None:
-        raise ValueError("Kolom jumlah SMA tidak ditemukan di file SMA.")
+        raise ValueError("Kolom jumlah SMA tidak ditemukan di file populasi provinsi.")
 
     meta_sma_df = pd.DataFrame({
         "Provinsi_Label": meta_sma_raw[col_prov_sma].astype(str).str.strip(),
@@ -360,7 +407,7 @@ try:
     extra_info.append(f"Meta SMA terbaca: **{META_SMA_FILE}** ({len(meta_sma_df)} provinsi)")
 except Exception as e:
     meta_sma_df = None
-    extra_info.append(f"⚠️ Gagal memuat file SMA: {e}")
+    extra_info.append(f"⚠️ Gagal memuat file populasi provinsi: {e}")
 
 if extra_info:
     DATA_INFO = DATA_INFO + "<br>" + "<br>".join(extra_info)
@@ -429,7 +476,7 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
         tmp_umum = tmp[tmp["_dataset"] == "umum"].copy() if "_dataset" in tmp.columns else tmp.copy()
         g_umum = tmp_umum.groupby("kab_key").size().rename("Sampel Umum").reset_index()
 
-        use_cols = ["kab_key", "Kab_Kota_Label", "Jml_Kecamatan", "Jml_DesaKel", "Jml_SD", "Jml_SMP"]
+        use_cols = ["kab_key", "Kab_Kota_Label", "Jml_Kecamatan", "Jml_DesaKel", "Jml_SD", "Jml_SMP", "Pop_Umum_Meta", "Pop_Sekolah_Meta"]
         use_cols = [c for c in use_cols if c in meta_kab_df.columns]
 
         merged = (
@@ -443,11 +490,20 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
             if c in merged.columns:
                 merged[c] = merged[c].fillna(0).astype(int)
 
-        merged["Populasi Sekolah (SD+SMP)"] = merged[["Jml_SD", "Jml_SMP"]].sum(axis=1, skipna=True)
-        merged["Populasi Admin (Kec+Desa/Kel)"] = merged.get("Jml_Kecamatan", np.nan) + merged.get("Jml_DesaKel", np.nan)
+        # Populasi sekolah: prefer SD+SMP, fallback ke Pop_Sekolah_Meta
+        pop_sek = merged[["Jml_SD", "Jml_SMP"]].sum(axis=1, skipna=True)
+        if "Pop_Sekolah_Meta" in merged.columns:
+            pop_sek = pop_sek.where(pop_sek.notna() & (pop_sek > 0), merged["Pop_Sekolah_Meta"])
+        merged["Populasi Sekolah (SD+SMP)"] = pop_sek
+
+        # Populasi umum: prefer Kec+Desa, fallback ke Pop_Umum_Meta
+        pop_umum = merged.get("Jml_Kecamatan", np.nan) + merged.get("Jml_DesaKel", np.nan)
+        if "Pop_Umum_Meta" in merged.columns:
+            pop_umum = pop_umum.where(pop_umum.notna() & (pop_umum > 0), merged["Pop_Umum_Meta"])
+        merged["Populasi Admin (Kec+Desa/Kel)"] = pop_umum
 
         merged["Target Sekolah (68%)"] = np.ceil(merged["Populasi Sekolah (SD+SMP)"] * TARGET_COVERAGE)
-        merged["Target Umum (68%)"] = np.ceil(merged["Populasi Admin (Kec+Desa/Kel)"] * TARGET_COVERAGE)
+        merged["Target Umum (68%)"]    = np.ceil(merged["Populasi Admin (Kec+Desa/Kel)"] * TARGET_COVERAGE)
 
         merged["Kekurangan Sampel Sekolah"] = merged.apply(
             lambda r: max(int(r["Target Sekolah (68%)"] - r["Sampel Sekolah"]) if pd.notna(r["Target Sekolah (68%)"]) else 0, 0),
@@ -487,7 +543,6 @@ def compute_gap_verification(df_filtered: pd.DataFrame, kew_value: str) -> pd.Da
 
         tmp["prov_key"] = tmp["prov_clean"].apply(norm_prov_label)
 
-        # start dari sampel (tidak bocor prov lain)
         g_total = tmp.groupby("prov_key").size().rename("Sampel Total (Prov)").reset_index()
 
         tmp_sek = tmp[tmp["_dataset"] == "sekolah"].copy() if "_dataset" in tmp.columns else tmp.copy()
@@ -742,7 +797,6 @@ def generate_word_report_gap(verif_df: pd.DataFrame, prov: str, kab: str, kew: s
     if not HAS_KALEIDO:
         doc.add_paragraph("Grafik pie tidak dibuat karena 'kaleido' tidak tersedia di server.")
     else:
-        # buat pie total capai vs target kalau ada pasangan kolom sampel-target
         pie_made = False
         if "Sampel Sekolah" in verif_df.columns and "Target Sekolah (68%)" in verif_df.columns:
             samp = pd.to_numeric(verif_df["Sampel Sekolah"], errors="coerce").fillna(0).sum()
@@ -878,9 +932,8 @@ Aplikasi ini menghitung **berapa unit lagi yang harus dikumpulkan** agar memenuh
 
 **File:**
 - `{DATA_FILE}` (DM)
-- `{META_KAB_FILE}` (Kecamatan + Desa/Kel)
-- `{META_SDSMP_FILE}` (SD + SMP)
-- `{META_SMA_FILE}` (SMA)
+- `{META_KAB_FILE}` (Populasi Kab/Kota)
+- `{META_SMA_FILE}` (Populasi Provinsi)
 
 {DATA_INFO}
 """