Update app.py
Browse files
app.py
CHANGED
|
@@ -1,18 +1,18 @@
|
|
| 1 |
# -*- coding: utf-8 -*-
|
| 2 |
"""
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
* Bell curve FINAL: all + per jenis
|
| 12 |
"""
|
| 13 |
|
| 14 |
import os
|
| 15 |
import re
|
|
|
|
| 16 |
import tempfile
|
| 17 |
from pathlib import Path
|
| 18 |
|
|
@@ -22,22 +22,39 @@ import pandas as pd
|
|
| 22 |
import plotly.graph_objects as go
|
| 23 |
from sklearn.preprocessing import PowerTransformer
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
# ============================================================
|
| 26 |
-
# 1) KONFIGURASI FILE
|
| 27 |
# ============================================================
|
| 28 |
|
| 29 |
-
DATA_FILE = "IPLM_clean_manual_131225.xlsx"
|
| 30 |
-
POP_KAB = "Data_populasi_Kab_kota.xlsx"
|
| 31 |
-
POP_PROV = "Data_populasi_propinsi.xlsx"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
|
| 37 |
# ============================================================
|
| 38 |
# 2) UTIL
|
| 39 |
# ============================================================
|
| 40 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
def _canon(s: str) -> str:
|
| 42 |
return re.sub(r"[^a-z0-9]+", "", str(s).lower())
|
| 43 |
|
|
@@ -48,9 +65,11 @@ def _disp_text(x):
|
|
| 48 |
return " ".join(t.split())
|
| 49 |
|
| 50 |
def pick_col(df, candidates):
|
|
|
|
| 51 |
for c in candidates:
|
| 52 |
if c in df.columns:
|
| 53 |
return c
|
|
|
|
| 54 |
can_map = {_canon(c): c for c in df.columns}
|
| 55 |
for c in candidates:
|
| 56 |
k = _canon(c)
|
|
@@ -125,10 +144,17 @@ def safe_div(num, den):
|
|
| 125 |
return float(num) / float(den)
|
| 126 |
|
| 127 |
def cap_bobot(cov: float) -> float:
|
|
|
|
| 128 |
if cov is None or pd.isna(cov) or cov <= 0:
|
| 129 |
-
return
|
| 130 |
return float(min(cov / TARGET_COVERAGE, 1.0))
|
| 131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
def penalized_mean(row, cols):
|
| 133 |
vals = []
|
| 134 |
for c in cols:
|
|
@@ -140,11 +166,6 @@ def penalized_mean(row, cols):
|
|
| 140 |
vals.append(float(v))
|
| 141 |
return float(np.mean(vals)) if vals else 0.0
|
| 142 |
|
| 143 |
-
def slugify(s: str) -> str:
|
| 144 |
-
if s is None:
|
| 145 |
-
return "NA"
|
| 146 |
-
t = str(s).strip()
|
| 147 |
-
return re.sub(r"[^A-Z0-9]+", "", t.upper()) or "NA"
|
| 148 |
|
| 149 |
# ============================================================
|
| 150 |
# 3) INDIKATOR IPLM
|
|
@@ -201,98 +222,9 @@ alias_map_raw = {
|
|
| 201 |
}
|
| 202 |
alias_map = {_canon(k): v for k, v in alias_map_raw.items()}
|
| 203 |
|
| 204 |
-
# ============================================================
|
| 205 |
-
# 4) LOAD DATA
|
| 206 |
-
# ============================================================
|
| 207 |
-
|
| 208 |
-
DATA_INFO = ""
|
| 209 |
-
df_all_raw = None
|
| 210 |
-
df_pop_kab = None
|
| 211 |
-
df_pop_prov = None
|
| 212 |
-
|
| 213 |
-
prov_col = kab_col = kew_col = jenis_col = nama_col = None
|
| 214 |
-
|
| 215 |
-
# --- DM ---
|
| 216 |
-
try:
|
| 217 |
-
fp = Path(DATA_FILE)
|
| 218 |
-
if not fp.exists():
|
| 219 |
-
raise FileNotFoundError(f"File tidak ditemukan: {DATA_FILE}")
|
| 220 |
-
|
| 221 |
-
xls = pd.ExcelFile(fp)
|
| 222 |
-
frames = [pd.read_excel(fp, sheet_name=s) for s in xls.sheet_names]
|
| 223 |
-
df_all_raw = pd.concat(frames, ignore_index=True, sort=False)
|
| 224 |
-
|
| 225 |
-
prov_col = pick_col(df_all_raw, ["provinsi", "Provinsi", "PROVINSI"])
|
| 226 |
-
kab_col = pick_col(df_all_raw, ["kab_kota", "Kab_Kota", "Kab/Kota", "KAB/KOTA", "kabupaten_kota"])
|
| 227 |
-
kew_col = pick_col(df_all_raw, ["kewenangan", "jenis_kewenangan", "Kewenangan", "KEWENANGAN"])
|
| 228 |
-
jenis_col = pick_col(df_all_raw, ["jenis_perpustakaan", "Jenis Perpustakaan", "JENIS_PERPUSTAKAAN"])
|
| 229 |
-
nama_col = pick_col(df_all_raw, ["nm_perpustakaan","nama_perpustakaan", "Nama Perpustakaan", "nm_instansi_lembaga"])
|
| 230 |
-
|
| 231 |
-
df_all_raw["KEW_NORM"] = df_all_raw[kew_col].apply(norm_kew) if kew_col else None
|
| 232 |
-
|
| 233 |
-
val_map_jenis = {
|
| 234 |
-
"PERPUSTAKAAN SEKOLAH": "sekolah", "SEKOLAH": "sekolah",
|
| 235 |
-
"PERPUSTAKAAN UMUM": "umum", "UMUM": "umum", "PERPUSTAKAAN DAERAH": "umum",
|
| 236 |
-
"PERPUSTAKAAN KHUSUS": "khusus", "KHUSUS": "khusus",
|
| 237 |
-
}
|
| 238 |
-
df_all_raw["_dataset"] = df_all_raw[jenis_col].astype(str).str.strip().str.upper().map(val_map_jenis) if jenis_col else None
|
| 239 |
-
|
| 240 |
-
df_all_raw["PROV_DISP"] = df_all_raw[prov_col].apply(_disp_text) if prov_col else None
|
| 241 |
-
df_all_raw["KAB_DISP"] = df_all_raw[kab_col].apply(_disp_text) if kab_col else None
|
| 242 |
-
|
| 243 |
-
DATA_INFO = f"β
DM terbaca: **{DATA_FILE}** | Baris: **{len(df_all_raw)}**"
|
| 244 |
-
except Exception as e:
|
| 245 |
-
df_all_raw = None
|
| 246 |
-
DATA_INFO = f"β οΈ Gagal memuat DM: `{e}`"
|
| 247 |
-
|
| 248 |
-
# --- Pop Kab/Kota ---
|
| 249 |
-
POP_INFO = []
|
| 250 |
-
try:
|
| 251 |
-
pk = pd.read_excel(POP_KAB)
|
| 252 |
-
c_prov = pick_col(pk, ["PROVINSI","Provinsi"])
|
| 253 |
-
c_kab = pick_col(pk, ["KABUPATEN_KOTA","Kab/Kota","Kabupaten/Kota","KAB/KOTA"])
|
| 254 |
-
c_pop_umum = pick_col(pk, ["Pop_Umum","pop_umum","jumlah_populasi_umum"])
|
| 255 |
-
c_pop_sekolah = pick_col(pk, ["Pop_Sekolah","pop_sekolah","jumlah_populasi_sekolah"])
|
| 256 |
-
|
| 257 |
-
if c_kab is None:
|
| 258 |
-
raise ValueError("Kolom Kab/Kota tidak ditemukan di populasi kab/kota.")
|
| 259 |
-
|
| 260 |
-
df_pop_kab = pd.DataFrame({
|
| 261 |
-
"Provinsi_Label": pk[c_prov].astype(str).str.strip() if c_prov else None,
|
| 262 |
-
"Kab_Kota_Label": pk[c_kab].astype(str).str.strip(),
|
| 263 |
-
"Pop_Umum": pk[c_pop_umum].apply(coerce_num) if c_pop_umum else np.nan,
|
| 264 |
-
"Pop_Sekolah": pk[c_pop_sekolah].apply(coerce_num) if c_pop_sekolah else np.nan,
|
| 265 |
-
})
|
| 266 |
-
df_pop_kab["kab_key"] = df_pop_kab["Kab_Kota_Label"].apply(norm_kab_label)
|
| 267 |
-
POP_INFO.append(f"β
Populasi Kab/Kota terbaca: **{POP_KAB}** (n={len(df_pop_kab)})")
|
| 268 |
-
except Exception as e:
|
| 269 |
-
df_pop_kab = None
|
| 270 |
-
POP_INFO.append(f"β οΈ Gagal memuat populasi Kab/Kota: `{e}`")
|
| 271 |
-
|
| 272 |
-
# --- Pop Provinsi ---
|
| 273 |
-
try:
|
| 274 |
-
pp = pd.read_excel(POP_PROV)
|
| 275 |
-
c_prov = pick_col(pp, ["Provinsi","PROVINSI"])
|
| 276 |
-
c_total = pick_col(pp, ["total_pend","TOTAL_PEND","Pop_Sekolah_Prov","pop_sekolah_prov","sma"])
|
| 277 |
-
if c_prov is None or c_total is None:
|
| 278 |
-
raise ValueError("Kolom Provinsi / total_pend (atau ekuivalen) tidak ditemukan di populasi provinsi.")
|
| 279 |
-
|
| 280 |
-
df_pop_prov = pd.DataFrame({
|
| 281 |
-
"Provinsi_Label": pp[c_prov].astype(str).str.strip(),
|
| 282 |
-
"Pop_Sekolah_Prov": pp[c_total].apply(coerce_num),
|
| 283 |
-
})
|
| 284 |
-
df_pop_prov["prov_key"] = df_pop_prov["Provinsi_Label"].apply(norm_prov_label)
|
| 285 |
-
df_pop_prov = df_pop_prov.groupby("prov_key", as_index=False).agg({"Provinsi_Label":"first","Pop_Sekolah_Prov":"sum"})
|
| 286 |
-
POP_INFO.append(f"β
Populasi Provinsi terbaca: **{POP_PROV}** (n={len(df_pop_prov)})")
|
| 287 |
-
except Exception as e:
|
| 288 |
-
df_pop_prov = None
|
| 289 |
-
POP_INFO.append(f"β οΈ Gagal memuat populasi Provinsi: `{e}`")
|
| 290 |
-
|
| 291 |
-
if POP_INFO:
|
| 292 |
-
DATA_INFO = DATA_INFO + "<br>" + "<br>".join(POP_INFO)
|
| 293 |
|
| 294 |
# ============================================================
|
| 295 |
-
#
|
| 296 |
# ============================================================
|
| 297 |
|
| 298 |
def prepare_global(df_src: pd.DataFrame) -> pd.DataFrame:
|
|
@@ -300,6 +232,7 @@ def prepare_global(df_src: pd.DataFrame) -> pd.DataFrame:
|
|
| 300 |
return df_src
|
| 301 |
df = df_src.copy()
|
| 302 |
|
|
|
|
| 303 |
rename_map = {}
|
| 304 |
for col in df.columns:
|
| 305 |
c = _canon(col)
|
|
@@ -314,9 +247,11 @@ def prepare_global(df_src: pd.DataFrame) -> pd.DataFrame:
|
|
| 314 |
df = df.rename(columns=rename_map)
|
| 315 |
|
| 316 |
available = [c for c in all_indicators if c in df.columns]
|
|
|
|
| 317 |
for c in available:
|
| 318 |
df[c] = df[c].apply(coerce_num)
|
| 319 |
|
|
|
|
| 320 |
for c in available:
|
| 321 |
x = df[c].astype(float).values
|
| 322 |
mask = ~np.isnan(x)
|
|
@@ -328,27 +263,163 @@ def prepare_global(df_src: pd.DataFrame) -> pd.DataFrame:
|
|
| 328 |
transformed[mask] = x[mask]
|
| 329 |
df[f"norm_{c}"] = minmax_norm(pd.Series(transformed, index=df.index))
|
| 330 |
|
| 331 |
-
df["sub_koleksi"]
|
| 332 |
-
df["sub_sdm"]
|
| 333 |
-
df["sub_pelayanan"]
|
| 334 |
df["sub_pengelolaan"] = df.apply(lambda r: penalized_mean(r, [c for c in pengelolaan_cols if c in available]), axis=1)
|
| 335 |
|
| 336 |
df["dim_kepatuhan"] = df[["sub_koleksi","sub_sdm"]].mean(axis=1)
|
| 337 |
df["dim_kinerja"] = df[["sub_pelayanan","sub_pengelolaan"]].mean(axis=1)
|
| 338 |
|
| 339 |
df["Indeks_Real_0_100"] = 100 * (W_KEPATUHAN * df["dim_kepatuhan"] + W_KINERJA * df["dim_kinerja"])
|
|
|
|
| 340 |
for c in ["sub_koleksi","sub_sdm","sub_pelayanan","sub_pengelolaan","dim_kepatuhan","dim_kinerja","Indeks_Real_0_100"]:
|
| 341 |
df[c] = df[c].fillna(0.0)
|
| 342 |
|
| 343 |
return df
|
| 344 |
|
| 345 |
-
df_all = prepare_global(df_all_raw) if df_all_raw is not None else None
|
| 346 |
|
| 347 |
# ============================================================
|
| 348 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 349 |
# ============================================================
|
| 350 |
|
| 351 |
-
def
|
| 352 |
if df_filtered is None or df_filtered.empty:
|
| 353 |
return df_filtered, pd.DataFrame()
|
| 354 |
|
|
@@ -357,15 +428,22 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 357 |
|
| 358 |
df["bobot_coverage"] = 1.0
|
| 359 |
df["coverage"] = np.nan
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
|
| 361 |
-
|
|
|
|
| 362 |
tmp = df.copy()
|
| 363 |
tmp["kab_key"] = tmp["KAB_DISP"].apply(norm_kab_label)
|
| 364 |
|
| 365 |
g = tmp.groupby(["kab_key","_dataset"]).size().rename("n_sampel").reset_index()
|
| 366 |
g_piv = g.pivot(index="kab_key", columns="_dataset", values="n_sampel").fillna(0)
|
| 367 |
|
| 368 |
-
pop =
|
| 369 |
|
| 370 |
rows = []
|
| 371 |
for kk in g_piv.index:
|
|
@@ -400,8 +478,10 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 400 |
})
|
| 401 |
|
| 402 |
verif_df = pd.DataFrame(rows)
|
|
|
|
|
|
|
|
|
|
| 403 |
|
| 404 |
-
# bulatkan TANPA koma
|
| 405 |
int_cols = ["Pop_Sekolah","Sampel_Sekolah","GAP_Ke_68_Sekolah","Pop_Umum","Sampel_Umum","GAP_Ke_68_Umum"]
|
| 406 |
pct_cols = ["Coverage_Sekolah_%","Bobot_Sekolah_68_%","Coverage_Umum_%","Bobot_Umum_68_%"]
|
| 407 |
for c in int_cols:
|
|
@@ -411,11 +491,11 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 411 |
if c in verif_df.columns:
|
| 412 |
verif_df[c] = verif_df[c].fillna(0).round(0).astype(int)
|
| 413 |
|
| 414 |
-
bobot_map_sek = {norm_kab_label(r["Kab/Kota"]): float(r["Bobot_Sekolah_68_%"]) / 100.0 for _, r in verif_df.iterrows()}
|
| 415 |
-
bobot_map_um = {norm_kab_label(r["Kab/Kota"]): float(r["Bobot_Umum_68_%"]) / 100.0 for _, r in verif_df.iterrows()}
|
| 416 |
|
| 417 |
-
cov_map_sek = {norm_kab_label(r["Kab/Kota"]): float(r["Coverage_Sekolah_%"]) / 100.0 for _, r in verif_df.iterrows()}
|
| 418 |
-
cov_map_um = {norm_kab_label(r["Kab/Kota"]): float(r["Coverage_Umum_%"]) / 100.0 for _, r in verif_df.iterrows()}
|
| 419 |
|
| 420 |
df["kab_key"] = df["KAB_DISP"].apply(norm_kab_label)
|
| 421 |
|
|
@@ -425,9 +505,9 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 425 |
if ds == "khusus":
|
| 426 |
return 1.0
|
| 427 |
if ds == "sekolah":
|
| 428 |
-
return float(bobot_map_sek.get(kk,
|
| 429 |
if ds == "umum":
|
| 430 |
-
return float(bobot_map_um.get(kk,
|
| 431 |
return 1.0
|
| 432 |
|
| 433 |
def row_cov(r):
|
|
@@ -442,20 +522,23 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 442 |
df["bobot_coverage"] = df.apply(row_weight, axis=1)
|
| 443 |
df["coverage"] = df.apply(row_cov, axis=1)
|
| 444 |
|
| 445 |
-
|
|
|
|
| 446 |
tmp = df.copy()
|
| 447 |
tmp["prov_key"] = tmp["PROV_DISP"].apply(norm_prov_label)
|
| 448 |
|
| 449 |
g = tmp.groupby(["prov_key","_dataset"]).size().rename("n_sampel").reset_index()
|
| 450 |
g_piv = g.pivot(index="prov_key", columns="_dataset", values="n_sampel").fillna(0)
|
| 451 |
-
pop =
|
| 452 |
|
| 453 |
rows = []
|
| 454 |
for pk in g_piv.index:
|
| 455 |
pop_sek = pop.loc[pk, "Pop_Sekolah_Prov"] if pk in pop.index else np.nan
|
| 456 |
n_sek = float(g_piv.loc[pk].get("sekolah", 0))
|
|
|
|
| 457 |
cov_sek = safe_div(n_sek, pop_sek)
|
| 458 |
bobot_sek = cap_bobot(cov_sek)
|
|
|
|
| 459 |
target_sek = (TARGET_COVERAGE * pop_sek) if not pd.isna(pop_sek) else np.nan
|
| 460 |
|
| 461 |
rows.append({
|
|
@@ -468,6 +551,8 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 468 |
})
|
| 469 |
|
| 470 |
verif_df = pd.DataFrame(rows)
|
|
|
|
|
|
|
| 471 |
|
| 472 |
int_cols = ["Pop_Sekolah","Sampel_Sekolah","GAP_Ke_68_Sekolah"]
|
| 473 |
pct_cols = ["Coverage_Sekolah_%","Bobot_Sekolah_68_%"]
|
|
@@ -478,8 +563,8 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 478 |
if c in verif_df.columns:
|
| 479 |
verif_df[c] = verif_df[c].fillna(0).round(0).astype(int)
|
| 480 |
|
| 481 |
-
bobot_map = {norm_prov_label(r["Provinsi"]): float(r["Bobot_Sekolah_68_%"]) / 100.0 for _, r in verif_df.iterrows()}
|
| 482 |
-
cov_map = {norm_prov_label(r["Provinsi"]): float(r["Coverage_Sekolah_%"]) / 100.0 for _, r in verif_df.iterrows()}
|
| 483 |
|
| 484 |
df["prov_key"] = df["PROV_DISP"].apply(norm_prov_label)
|
| 485 |
|
|
@@ -488,7 +573,7 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 488 |
if ds == "khusus":
|
| 489 |
return 1.0
|
| 490 |
if ds == "sekolah":
|
| 491 |
-
return float(bobot_map.get(r.get("prov_key", None),
|
| 492 |
return 1.0
|
| 493 |
|
| 494 |
def row_cov(r):
|
|
@@ -499,57 +584,118 @@ def compute_final(df_filtered: pd.DataFrame, kew_value: str):
|
|
| 499 |
df["bobot_coverage"] = df.apply(row_weight, axis=1)
|
| 500 |
df["coverage"] = df.apply(row_cov, axis=1)
|
| 501 |
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
df["Indeks_Final_0_100"] = (df["Indeks_Real_0_100"].fillna(0.0) * df["bobot_coverage"].fillna(0.0)).fillna(0.0)
|
| 506 |
return df, verif_df
|
| 507 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 508 |
# ============================================================
|
| 509 |
-
#
|
| 510 |
# ============================================================
|
| 511 |
|
| 512 |
-
def make_bell_figure(
|
| 513 |
fig = go.Figure()
|
| 514 |
-
|
| 515 |
-
|
|
|
|
| 516 |
return fig
|
| 517 |
|
| 518 |
-
dfp =
|
| 519 |
if dfp.empty or len(dfp) < min_points:
|
| 520 |
-
fig.
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
yaxis_title="Kepadatan (relatif)",
|
| 524 |
-
annotations=[dict(text="Grafik tidak ditampilkan (data terlalu sedikit).",
|
| 525 |
-
x=0.5, y=0.5, xref="paper", yref="paper",
|
| 526 |
-
showarrow=False, font=dict(size=14))]
|
| 527 |
)
|
| 528 |
return fig
|
| 529 |
|
| 530 |
-
|
| 531 |
-
mu = float(np.mean(
|
| 532 |
-
sigma = float(np.std(
|
| 533 |
-
|
| 534 |
-
sigma = 1.0
|
| 535 |
|
| 536 |
-
xs = np.linspace(max(0, np.min(
|
| 537 |
pdf = (1.0 / (sigma * np.sqrt(2 * np.pi))) * np.exp(-0.5 * ((xs - mu) / sigma) ** 2)
|
| 538 |
-
pdf = pdf /
|
| 539 |
|
| 540 |
if name_col and name_col in dfp.columns:
|
| 541 |
-
|
| 542 |
else:
|
| 543 |
-
|
| 544 |
|
| 545 |
fig.add_trace(go.Scatter(x=xs, y=pdf, mode="lines", name="Bell curve", hoverinfo="skip"))
|
| 546 |
-
fig.add_trace(go.Scatter(
|
| 547 |
-
|
| 548 |
-
mode="markers", name="Perpustakaan",
|
| 549 |
-
hovertext=hover_text, hovertemplate="%{hovertext}<extra></extra>"
|
| 550 |
-
))
|
| 551 |
|
| 552 |
-
q1, q2, q3 = np.quantile(
|
| 553 |
for q, label in [(q1, "Q1"), (q2, "Q2 (Median)"), (q3, "Q3")]:
|
| 554 |
fig.add_trace(go.Scatter(
|
| 555 |
x=[q, q], y=[0, 1.05],
|
|
@@ -558,546 +704,356 @@ def make_bell_figure(df_in: pd.DataFrame, title: str, index_col="Indeks_Final_0_
|
|
| 558 |
))
|
| 559 |
|
| 560 |
fig.update_layout(
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
yaxis_title="Kepadatan (relatif)",
|
| 564 |
-
yaxis=dict(showticklabels=False, zeroline=True, range=[0, 1.2]),
|
| 565 |
margin=dict(l=40, r=20, t=60, b=40),
|
| 566 |
hovermode="x"
|
| 567 |
)
|
| 568 |
return fig
|
| 569 |
|
|
|
|
| 570 |
# ============================================================
|
| 571 |
-
#
|
| 572 |
-
# (TAMBAHAN SAJA β TIDAK MENGUBAH PIPELINE YANG ADA)
|
| 573 |
# ============================================================
|
| 574 |
|
| 575 |
-
|
| 576 |
-
if df is None or df.empty:
|
| 577 |
-
return "(kosong)"
|
| 578 |
-
tmp = df.copy()
|
| 579 |
-
# batasi kolom & baris biar prompt tidak meledak
|
| 580 |
-
tmp = tmp.head(max_rows)
|
| 581 |
-
return tmp.to_string(index=False)
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
def summarize_distribution(detail_df: pd.DataFrame):
|
| 585 |
-
"""
|
| 586 |
-
Ringkas distribusi indeks final untuk LLM:
|
| 587 |
-
- pakai Indeks_Final_0_100 kalau ada, kalau tidak fallback ke Indeks_Real_0_100
|
| 588 |
-
"""
|
| 589 |
-
idx_col = "Indeks_Final_0_100" if (detail_df is not None and "Indeks_Final_0_100" in detail_df.columns) else "Indeks_Real_0_100"
|
| 590 |
-
if detail_df is None or detail_df.empty or idx_col not in detail_df.columns:
|
| 591 |
-
return {"idx_col": idx_col, "all": {}, "by_type": {}}
|
| 592 |
-
|
| 593 |
-
out = {"idx_col": idx_col, "all": {}, "by_type": {}}
|
| 594 |
-
|
| 595 |
-
def stats_for(s: pd.Series):
|
| 596 |
-
s = pd.to_numeric(s, errors="coerce").dropna()
|
| 597 |
-
if len(s) == 0:
|
| 598 |
-
return {}
|
| 599 |
-
q1, q2, q3 = np.quantile(s.values, [0.25, 0.5, 0.75])
|
| 600 |
-
return {
|
| 601 |
-
"n": int(len(s)),
|
| 602 |
-
"mean": float(s.mean()),
|
| 603 |
-
"std": float(s.std(ddof=1)) if len(s) > 1 else 0.0,
|
| 604 |
-
"min": float(s.min()),
|
| 605 |
-
"q1": float(q1),
|
| 606 |
-
"median": float(q2),
|
| 607 |
-
"q3": float(q3),
|
| 608 |
-
"max": float(s.max()),
|
| 609 |
-
}
|
| 610 |
-
|
| 611 |
-
out["all"] = stats_for(detail_df[idx_col])
|
| 612 |
-
|
| 613 |
-
if "_dataset" in detail_df.columns:
|
| 614 |
-
for ds in ["sekolah", "umum", "khusus"]:
|
| 615 |
-
dsub = detail_df[detail_df["_dataset"] == ds]
|
| 616 |
-
out["by_type"][ds] = stats_for(dsub[idx_col])
|
| 617 |
-
|
| 618 |
-
return out
|
| 619 |
-
|
| 620 |
|
| 621 |
-
def
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
"""
|
| 632 |
-
wilayah = kab_name
|
| 633 |
-
if kew_value and kew_value != "(Semua)":
|
| 634 |
-
wilayah = f"{kab_name} (kewenangan {kew_value})"
|
| 635 |
-
|
| 636 |
-
dist = summarize_distribution(detail_df)
|
| 637 |
-
idx_col = dist.get("idx_col", "Indeks_Final_0_100")
|
| 638 |
-
|
| 639 |
-
# ringkas angka utama biar prompt padat
|
| 640 |
-
all_stats = dist.get("all", {})
|
| 641 |
-
by_type = dist.get("by_type", {})
|
| 642 |
-
|
| 643 |
-
def fmt_stats(d):
|
| 644 |
-
if not d:
|
| 645 |
-
return "(tidak tersedia)"
|
| 646 |
-
return (
|
| 647 |
-
f"n={d['n']}, mean={d['mean']:.2f}, sd={d['std']:.2f}, "
|
| 648 |
-
f"min={d['min']:.2f}, Q1={d['q1']:.2f}, median={d['median']:.2f}, Q3={d['q3']:.2f}, max={d['max']:.2f}"
|
| 649 |
-
)
|
| 650 |
|
|
|
|
| 651 |
lines = []
|
| 652 |
lines.append(f"Wilayah: {wilayah}")
|
| 653 |
-
lines.append(f"
|
| 654 |
-
lines.append(f"
|
| 655 |
-
if
|
| 656 |
-
|
| 657 |
-
|
|
|
|
|
|
|
| 658 |
|
| 659 |
-
|
| 660 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 661 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 662 |
client = get_llm_client()
|
| 663 |
if client is None or not USE_LLM:
|
| 664 |
-
|
| 665 |
-
rb = generate_rule_based_analysis(detail_df, agg_df, kab_name, kew_value)
|
| 666 |
-
return (
|
| 667 |
-
"β οΈ LLM tidak tersedia, analisis menggunakan rule-based.\n\n" + rb
|
| 668 |
-
)
|
| 669 |
|
| 670 |
system_prompt = (
|
| 671 |
-
"Anda adalah analis
|
| 672 |
-
"Anda
|
| 673 |
-
"Anda harus menggunakan pendekatan berbasis data, jelas, dan ringkas."
|
| 674 |
)
|
| 675 |
-
|
| 676 |
user_prompt = f"""
|
| 677 |
-
DATA RINGKAS IPLM (
|
| 678 |
-
|
| 679 |
-
RINGKASAN STATISTIK (indeks final & distribusi):
|
| 680 |
-
{chr(10).join(lines)}
|
| 681 |
-
|
| 682 |
-
TABEL AGREGAT (ringkas):
|
| 683 |
-
{agg_txt}
|
| 684 |
|
| 685 |
-
|
| 686 |
-
{ver_txt}
|
| 687 |
|
| 688 |
-
|
| 689 |
-
|
|
|
|
|
|
|
|
|
|
| 690 |
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
C. Prioritas intervensi 12β18 bulan (1β2 paragraf) β fokus pada program pembinaan yang realistis.
|
| 696 |
-
D. Rekomendasi kebijakan 3β5 tahun (1β2 paragraf) β penataan tata kelola data, pembinaan, standardisasi.
|
| 697 |
-
|
| 698 |
-
GAYA:
|
| 699 |
-
- Jangan menyebut "rendah/sedang/tinggi". Gunakan frasa netral: "ruang penguatan", "belum konsisten", dll.
|
| 700 |
-
- Hindari kalimat terlalu panjang.
|
| 701 |
-
- Jangan membuat data baru di luar yang tersedia.
|
| 702 |
"""
|
| 703 |
-
|
| 704 |
try:
|
| 705 |
resp = client.chat_completion(
|
| 706 |
model=LLM_MODEL_NAME,
|
| 707 |
-
messages=[
|
| 708 |
-
|
| 709 |
-
{"role": "user", "content": user_prompt},
|
| 710 |
-
],
|
| 711 |
-
max_tokens=1200,
|
| 712 |
temperature=0.25,
|
| 713 |
top_p=0.9,
|
| 714 |
)
|
| 715 |
text = resp.choices[0].message.content.strip()
|
| 716 |
-
|
| 717 |
-
raise ValueError("Respon LLM kosong.")
|
| 718 |
-
return text
|
| 719 |
except Exception as e:
|
| 720 |
-
|
| 721 |
-
return (
|
| 722 |
-
"β οΈ Gagal memanggil LLM untuk data analytics, fallback rule-based.\n\n"
|
| 723 |
-
f"(Detail teknis: {repr(e)})\n\n{rb}"
|
| 724 |
-
)
|
| 725 |
-
|
| 726 |
|
| 727 |
-
def generate_word_report_llm_analytics(detail_df, agg_df, verif_df, prov, kab, kew, analytics_text):
|
| 728 |
-
"""
|
| 729 |
-
Word report yang menaruh:
|
| 730 |
-
- Ringkasan indeks FINAL (statistik & kuartil)
|
| 731 |
-
- Tabel agregat ringkas
|
| 732 |
-
- Tabel verifikasi coverage (dibulatkan TANPA koma)
|
| 733 |
-
- Narasi LLM data analytics
|
| 734 |
-
"""
|
| 735 |
-
if kew == "PUSAT":
|
| 736 |
-
return None
|
| 737 |
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
| 741 |
-
all_stats = dist.get("all", {})
|
| 742 |
|
|
|
|
|
|
|
| 743 |
doc = Document()
|
| 744 |
-
doc.add_heading(f"Laporan
|
| 745 |
-
doc.add_paragraph(
|
| 746 |
-
|
| 747 |
-
|
| 748 |
-
)
|
| 749 |
|
| 750 |
-
doc.add_heading("
|
| 751 |
-
if
|
| 752 |
-
doc.add_paragraph(f"
|
| 753 |
-
doc.add_paragraph(f"-
|
| 754 |
-
doc.add_paragraph(f"- Rata-rata: {all_stats.get('mean', 0.0):.2f}")
|
| 755 |
-
doc.add_paragraph(f"- Q1: {all_stats.get('q1', 0.0):.2f}")
|
| 756 |
-
doc.add_paragraph(f"- Median: {all_stats.get('median', 0.0):.2f}")
|
| 757 |
-
doc.add_paragraph(f"- Q3: {all_stats.get('q3', 0.0):.2f}")
|
| 758 |
-
doc.add_paragraph(f"- MinimumβMaksimum: {all_stats.get('min', 0.0):.2f} β {all_stats.get('max', 0.0):.2f}")
|
| 759 |
-
else:
|
| 760 |
-
doc.add_paragraph("Statistik distribusi tidak tersedia (data indeks tidak ditemukan).")
|
| 761 |
|
| 762 |
-
doc.add_heading("
|
| 763 |
if agg_df is not None and not agg_df.empty:
|
| 764 |
table = doc.add_table(rows=1, cols=len(agg_df.columns))
|
| 765 |
hdr = table.rows[0].cells
|
| 766 |
for i, c in enumerate(agg_df.columns):
|
| 767 |
hdr[i].text = str(c)
|
| 768 |
for _, row in agg_df.iterrows():
|
| 769 |
-
|
| 770 |
for i, c in enumerate(agg_df.columns):
|
| 771 |
-
|
| 772 |
else:
|
| 773 |
-
doc.add_paragraph("
|
| 774 |
|
| 775 |
-
doc.add_heading("
|
| 776 |
if verif_df is not None and not verif_df.empty:
|
| 777 |
-
|
| 778 |
-
|
| 779 |
-
# BULATKAN TANPA KOMa: semua numerik -> integer
|
| 780 |
-
for c in v.columns:
|
| 781 |
-
if pd.api.types.is_numeric_dtype(v[c]):
|
| 782 |
-
v[c] = pd.to_numeric(v[c], errors="coerce").fillna(0).round(0).astype(int)
|
| 783 |
-
|
| 784 |
-
table = doc.add_table(rows=1, cols=len(v.columns))
|
| 785 |
hdr = table.rows[0].cells
|
| 786 |
-
for i, c in enumerate(
|
| 787 |
hdr[i].text = str(c)
|
| 788 |
-
for _, row in
|
| 789 |
-
|
| 790 |
-
for i, c in enumerate(
|
| 791 |
-
|
| 792 |
else:
|
| 793 |
-
doc.add_paragraph("Tidak ada tabel verifikasi
|
| 794 |
|
| 795 |
-
doc.add_heading("
|
| 796 |
-
for
|
| 797 |
-
if
|
| 798 |
-
doc.add_paragraph(
|
| 799 |
|
| 800 |
outpath = tempfile.mktemp(suffix=".docx")
|
| 801 |
doc.save(outpath)
|
| 802 |
return outpath
|
| 803 |
|
|
|
|
| 804 |
# ============================================================
|
| 805 |
-
#
|
| 806 |
# ============================================================
|
| 807 |
|
| 808 |
-
def
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
|
| 812 |
-
|
| 813 |
-
|
| 814 |
-
|
| 815 |
-
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
|
| 819 |
-
|
| 820 |
-
|
| 821 |
-
|
| 822 |
-
"Rata2_Indeks_Final_0_100": float(sub["Indeks_Final_0_100"].mean()) if len(sub) else 0.0,
|
| 823 |
-
}
|
| 824 |
-
return row
|
| 825 |
-
|
| 826 |
-
for ds in ["sekolah","umum","khusus"]:
|
| 827 |
-
sub = df2[df2["_dataset"] == ds] if "_dataset" in df2.columns else df2.iloc[0:0]
|
| 828 |
-
rows.append(summarize(sub, label_map.get(ds, ds)))
|
| 829 |
-
|
| 830 |
-
rows.append(summarize(df2, "Rata-rata keseluruhan"))
|
| 831 |
-
return pd.DataFrame(rows).round(4)
|
| 832 |
-
|
| 833 |
-
def build_detail_ringkas(df2: pd.DataFrame, nama_col: str):
|
| 834 |
-
cols = ["PROV_DISP","KAB_DISP"]
|
| 835 |
-
if nama_col and nama_col in df2.columns:
|
| 836 |
-
cols.append(nama_col)
|
| 837 |
-
cols += ["KEW_NORM","_dataset",
|
| 838 |
-
"sub_koleksi","sub_sdm","sub_pelayanan","sub_pengelolaan",
|
| 839 |
-
"dim_kepatuhan","dim_kinerja",
|
| 840 |
-
"Indeks_Final_0_100"]
|
| 841 |
-
cols = [c for c in cols if c in df2.columns]
|
| 842 |
-
return df2[cols].copy().round(4)
|
| 843 |
|
| 844 |
-
|
| 845 |
-
# 9) PIPELINE FILTERED (DEDUP) + EXPORT + BELL CURVE
|
| 846 |
-
# ============================================================
|
| 847 |
|
| 848 |
-
|
| 849 |
-
|
| 850 |
-
|
| 851 |
-
|
| 852 |
-
|
| 853 |
-
|
| 854 |
-
|
| 855 |
-
df = df_all.copy()
|
| 856 |
-
|
| 857 |
-
if "PROV_DISP" in df.columns and prov_value and prov_value != "(Semua)":
|
| 858 |
-
df = df[df["PROV_DISP"] == prov_value]
|
| 859 |
-
if "KAB_DISP" in df.columns and kab_value and kab_value != "(Semua)":
|
| 860 |
-
df = df[df["KAB_DISP"] == kab_value]
|
| 861 |
-
if kew_value and kew_value != "(Semua)":
|
| 862 |
-
df = df[df["KEW_NORM"] == kew_value]
|
| 863 |
-
|
| 864 |
-
if df.empty:
|
| 865 |
-
return (pd.DataFrame(), pd.DataFrame(), pd.DataFrame(),
|
| 866 |
-
None, None, None,
|
| 867 |
-
go.Figure(), go.Figure(), go.Figure(), go.Figure(),
|
| 868 |
-
"Tidak ada data untuk kombinasi filter.")
|
| 869 |
-
|
| 870 |
-
df2, verif_df = compute_final(df, kew_value)
|
| 871 |
-
|
| 872 |
-
# DEDUP kunci (prov,kab,nama,kew,dataset)
|
| 873 |
-
kcols = [c for c in ["PROV_DISP","KAB_DISP","KEW_NORM","_dataset"] if c in df2.columns]
|
| 874 |
-
if nama_col and nama_col in df2.columns:
|
| 875 |
-
kcols.append(nama_col)
|
| 876 |
-
if kcols:
|
| 877 |
-
df2 = df2.drop_duplicates(subset=kcols, keep="first").copy()
|
| 878 |
-
|
| 879 |
-
agg_df = build_agg_ringkas(df2)
|
| 880 |
-
detail_df = build_detail_ringkas(df2, nama_col)
|
| 881 |
-
|
| 882 |
-
# Bell curves (FINAL)
|
| 883 |
-
ncol = nama_col if (nama_col and nama_col in df2.columns) else None
|
| 884 |
-
fig_all = make_bell_figure(df2, "Bell Curve Indeks FINAL β Semua Perpustakaan", name_col=ncol, min_points=5)
|
| 885 |
-
fig_sek = make_bell_figure(df2[df2["_dataset"]=="sekolah"], "Bell Curve Indeks FINAL β Perpustakaan Sekolah", name_col=ncol, min_points=3)
|
| 886 |
-
fig_um = make_bell_figure(df2[df2["_dataset"]=="umum"], "Bell Curve Indeks FINAL β Perpustakaan Umum", name_col=ncol, min_points=3)
|
| 887 |
-
fig_kh = make_bell_figure(df2[df2["_dataset"]=="khusus"], "Bell Curve Indeks FINAL β Perpustakaan Khusus", name_col=ncol, min_points=3)
|
| 888 |
-
|
| 889 |
-
tmpdir = tempfile.mkdtemp()
|
| 890 |
-
wilayah = kab_value if kab_value and kab_value != "(Semua)" else (prov_value if prov_value and prov_value != "(Semua)" else "NASIONAL")
|
| 891 |
-
slug = slugify(wilayah) + "_" + slugify(kew_value)
|
| 892 |
-
agg_path = os.path.join(tmpdir, f"IPLM_Agregat_RINGKAS_{slug}.xlsx")
|
| 893 |
-
detail_path = os.path.join(tmpdir, f"IPLM_Detail_RINGKAS_{slug}.xlsx")
|
| 894 |
-
verif_path = os.path.join(tmpdir, f"IPLM_VerifikasiCoverage_{slug}.xlsx")
|
| 895 |
-
|
| 896 |
-
agg_df.to_excel(agg_path, index=False)
|
| 897 |
-
detail_df.to_excel(detail_path, index=False)
|
| 898 |
-
(verif_df if verif_df is not None else pd.DataFrame()).to_excel(verif_path, index=False)
|
| 899 |
-
|
| 900 |
-
msg = f"β
Selesai. Unit (dedup): {len(df2)} | Wilayah: {wilayah} | Kew: {kew_value} | Mean Final: {df2['Indeks_Final_0_100'].mean():.2f}"
|
| 901 |
-
return agg_df, detail_df, verif_df, agg_path, detail_path, verif_path, fig_all, fig_sek, fig_um, fig_kh, msg
|
| 902 |
-
|
| 903 |
-
|
| 904 |
-
#===========================================================
|
| 905 |
-
# 9b. WRAPPER: PAKAI LLM DATA ANALYTICS + WORD (tanpa ubah run_app lama)
|
| 906 |
-
# ============================================================
|
| 907 |
|
| 908 |
-
if
|
| 909 |
-
|
| 910 |
-
def run_app(prov_value, kab_value, kew_value):
|
| 911 |
-
# jalankan versi asli dulu
|
| 912 |
-
(
|
| 913 |
-
agg_df,
|
| 914 |
-
detail_df_view,
|
| 915 |
-
verif_df,
|
| 916 |
-
agg_path,
|
| 917 |
-
detail_path,
|
| 918 |
-
raw_path,
|
| 919 |
-
word_path,
|
| 920 |
-
fig_all,
|
| 921 |
-
fig_sekolah,
|
| 922 |
-
fig_umum,
|
| 923 |
-
fig_khusus,
|
| 924 |
-
msg,
|
| 925 |
-
analysis_text,
|
| 926 |
-
) = _run_app_base(prov_value, kab_value, kew_value)
|
| 927 |
-
|
| 928 |
-
# kalau kosong, langsung return
|
| 929 |
-
if detail_df_view is None or (hasattr(detail_df_view, "empty") and detail_df_view.empty):
|
| 930 |
-
return (
|
| 931 |
-
agg_df, detail_df_view, verif_df,
|
| 932 |
-
agg_path, detail_path, raw_path,
|
| 933 |
-
word_path,
|
| 934 |
-
fig_all, fig_sekolah, fig_umum, fig_khusus,
|
| 935 |
-
msg,
|
| 936 |
-
analysis_text
|
| 937 |
-
)
|
| 938 |
|
| 939 |
-
|
| 940 |
-
|
| 941 |
-
|
| 942 |
-
if df is None or df.empty:
|
| 943 |
-
return (
|
| 944 |
-
agg_df, detail_df_view, verif_df,
|
| 945 |
-
agg_path, detail_path, raw_path,
|
| 946 |
-
word_path,
|
| 947 |
-
fig_all, fig_sekolah, fig_umum, fig_khusus,
|
| 948 |
-
msg,
|
| 949 |
-
analysis_text
|
| 950 |
-
)
|
| 951 |
|
| 952 |
-
|
| 953 |
-
df = df[df[prov_col_glob].astype(str).str.strip() == prov_value]
|
| 954 |
-
if kab_col_glob and kab_value and kab_value != "(Semua)":
|
| 955 |
-
df = df[df[kab_col_glob].astype(str).str.strip() == kab_value]
|
| 956 |
-
if kew_value and kew_value != "(Semua)":
|
| 957 |
-
df = df[df["KEW_NORM"] == kew_value]
|
| 958 |
|
| 959 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 960 |
return (
|
| 961 |
-
|
| 962 |
-
agg_path,
|
| 963 |
-
|
| 964 |
-
|
| 965 |
-
msg,
|
| 966 |
-
analysis_text
|
| 967 |
)
|
|
|
|
|
|
|
| 968 |
|
| 969 |
-
kab_name = kab_value if kab_value and kab_value != "(Semua)" else "SEMUA KAB/KOTA"
|
| 970 |
-
kew_name = kew_value if kew_value and kew_value != "(Semua)" else "SEMUA KEWENANGAN"
|
| 971 |
|
| 972 |
-
|
| 973 |
-
|
|
|
|
| 974 |
|
| 975 |
-
|
| 976 |
-
|
| 977 |
-
detail_df=detail_df_full,
|
| 978 |
-
agg_df=agg_df2 if (agg_df2 is not None and not agg_df2.empty) else agg_df,
|
| 979 |
-
verif_df=verif_df,
|
| 980 |
-
kab_name=kab_name,
|
| 981 |
-
kew_value=kew_value,
|
| 982 |
-
)
|
| 983 |
|
| 984 |
-
|
| 985 |
-
|
| 986 |
-
|
| 987 |
-
|
| 988 |
-
|
| 989 |
-
|
| 990 |
-
|
| 991 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 992 |
|
| 993 |
-
# Kembalikan output yang sama seperti run_app asli
|
| 994 |
return (
|
| 995 |
-
|
| 996 |
-
|
| 997 |
-
|
| 998 |
-
|
| 999 |
-
detail_path,
|
| 1000 |
-
raw_path,
|
| 1001 |
-
(word_path2 or word_path),
|
| 1002 |
-
fig_all,
|
| 1003 |
-
fig_sekolah,
|
| 1004 |
-
fig_umum,
|
| 1005 |
-
fig_khusus,
|
| 1006 |
-
msg,
|
| 1007 |
-
analytics_text # replace analysis_out dengan versi data analytics
|
| 1008 |
)
|
| 1009 |
|
| 1010 |
-
|
| 1011 |
-
#
|
| 1012 |
-
|
|
|
|
|
|
|
| 1013 |
|
| 1014 |
-
|
| 1015 |
-
|
| 1016 |
-
|
| 1017 |
-
|
| 1018 |
-
vals = sorted(list(dict.fromkeys([v for v in vals.tolist() if str(v).strip() != ""])))
|
| 1019 |
-
return ["(Semua)"] + vals
|
| 1020 |
-
|
| 1021 |
-
def get_kab_choices_for_prov(prov_value):
|
| 1022 |
-
if df_all_raw is None or "KAB_DISP" not in df_all_raw.columns:
|
| 1023 |
-
return ["(Semua)"]
|
| 1024 |
-
tmp = df_all_raw.copy()
|
| 1025 |
-
if prov_value and prov_value != "(Semua)":
|
| 1026 |
-
tmp = tmp[tmp["PROV_DISP"] == prov_value]
|
| 1027 |
-
vals = tmp["KAB_DISP"].dropna()
|
| 1028 |
-
vals = sorted(list(dict.fromkeys([v for v in vals.tolist() if str(v).strip() != ""])))
|
| 1029 |
-
return ["(Semua)"] + vals
|
| 1030 |
-
|
| 1031 |
-
def all_kew_choices():
|
| 1032 |
-
if df_all_raw is None or "KEW_NORM" not in df_all_raw.columns:
|
| 1033 |
-
return ["(Semua)"]
|
| 1034 |
-
vals = df_all_raw["KEW_NORM"].dropna().astype(str).str.strip()
|
| 1035 |
-
vals = sorted(list(dict.fromkeys([v for v in vals.tolist() if v != ""])))
|
| 1036 |
-
return ["(Semua)"] + (vals if vals else ["KAB/KOTA","PROVINSI"])
|
| 1037 |
-
|
| 1038 |
-
prov_choices = all_prov_choices()
|
| 1039 |
-
kab_choices = get_kab_choices_for_prov(prov_choices[0] if prov_choices else "(Semua)")
|
| 1040 |
-
kew_choices = all_kew_choices()
|
| 1041 |
-
default_kew = "KAB/KOTA" if "KAB/KOTA" in kew_choices else (kew_choices[1] if len(kew_choices) > 1 else "(Semua)")
|
| 1042 |
|
| 1043 |
-
|
| 1044 |
-
|
| 1045 |
-
return gr.update(choices=new_choices, value="(Semua)")
|
| 1046 |
|
| 1047 |
-
# ============================================================
|
| 1048 |
-
# 11) UI
|
| 1049 |
-
# ============================================================
|
| 1050 |
|
| 1051 |
with gr.Blocks() as demo:
|
| 1052 |
-
gr.Markdown(
|
| 1053 |
-
|
| 1054 |
-
# IPLM 2025 β Output Ringkas (Sub-dimensi + Dimensi + FINAL saja)
|
| 1055 |
-
**Final** sudah termasuk sanksi coverage 68% (internal).
|
| 1056 |
-
Verifikasi ditampilkan dalam integer (tanpa koma) agar bersih.
|
| 1057 |
|
| 1058 |
-
|
| 1059 |
-
|
| 1060 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1061 |
|
| 1062 |
with gr.Row():
|
| 1063 |
-
dd_prov = gr.Dropdown(label="Provinsi", choices=
|
| 1064 |
-
dd_kab = gr.Dropdown(label="Kab/Kota", choices=
|
| 1065 |
-
dd_kew = gr.Dropdown(label="Kewenangan", choices=
|
| 1066 |
|
| 1067 |
-
dd_prov.change(fn=on_prov_change, inputs=dd_prov, outputs=dd_kab)
|
| 1068 |
|
| 1069 |
run_btn = gr.Button("Jalankan Perhitungan")
|
| 1070 |
msg_out = gr.Markdown()
|
| 1071 |
|
| 1072 |
-
gr.Markdown("## Agregat (
|
| 1073 |
agg_out = gr.DataFrame(interactive=False)
|
| 1074 |
|
| 1075 |
-
gr.Markdown("## Detail (
|
| 1076 |
detail_out = gr.DataFrame(interactive=False)
|
| 1077 |
|
| 1078 |
-
gr.Markdown("## Verifikasi Coverage & GAP menuju 68% (kontrol mutu) β tanpa koma")
|
| 1079 |
verif_out = gr.DataFrame(interactive=False)
|
| 1080 |
|
| 1081 |
gr.Markdown("## Bell Curve Indeks FINAL β Semua Perpustakaan")
|
| 1082 |
bell_all = gr.Plot()
|
| 1083 |
|
| 1084 |
-
gr.Markdown("## Bell Curve Indeks FINAL β
|
| 1085 |
bell_sek = gr.Plot()
|
| 1086 |
-
bell_um = gr.Plot()
|
| 1087 |
-
bell_kh = gr.Plot()
|
| 1088 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1089 |
with gr.Row():
|
| 1090 |
-
|
| 1091 |
-
|
| 1092 |
-
|
|
|
|
| 1093 |
|
| 1094 |
run_btn.click(
|
| 1095 |
-
fn=
|
| 1096 |
-
inputs=[dd_prov, dd_kab, dd_kew],
|
| 1097 |
-
outputs=[
|
| 1098 |
-
|
| 1099 |
-
|
| 1100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1101 |
)
|
| 1102 |
|
| 1103 |
demo.launch()
|
|
|
|
| 1 |
# -*- coding: utf-8 -*-
|
| 2 |
"""
|
| 3 |
+
IPLM 2025 β FINAL (NO UPLOAD)
|
| 4 |
+
Penalti Coverage 68% + Bell Curve + Analisis LLM (Word)
|
| 5 |
+
|
| 6 |
+
FIX UTAMA:
|
| 7 |
+
1) Dropdown tidak error (callback tidak tergantung state None).
|
| 8 |
+
2) Download tanpa upload: gunakan gr.DownloadButton (bukan gr.File).
|
| 9 |
+
3) Cache loader berbasis mtime (hindari baca ulang).
|
| 10 |
+
4) Penalti coverage aman: populasi missing/0 -> bobot=1 (tanpa penalti).
|
|
|
|
| 11 |
"""
|
| 12 |
|
| 13 |
import os
|
| 14 |
import re
|
| 15 |
+
import time
|
| 16 |
import tempfile
|
| 17 |
from pathlib import Path
|
| 18 |
|
|
|
|
| 22 |
import plotly.graph_objects as go
|
| 23 |
from sklearn.preprocessing import PowerTransformer
|
| 24 |
|
| 25 |
+
from docx import Document
|
| 26 |
+
from huggingface_hub import InferenceClient
|
| 27 |
+
|
| 28 |
+
|
| 29 |
# ============================================================
|
| 30 |
+
# 1) KONFIGURASI FILE & PARAMETER
|
| 31 |
# ============================================================
|
| 32 |
|
| 33 |
+
DATA_FILE = os.getenv("DATA_FILE", "IPLM_clean_manual_131225.xlsx")
|
| 34 |
+
POP_KAB = os.getenv("POP_KAB", "Data_populasi_Kab_kota.xlsx")
|
| 35 |
+
POP_PROV = os.getenv("POP_PROV", "Data_populasi_propinsi.xlsx")
|
| 36 |
+
|
| 37 |
+
TARGET_COVERAGE = float(os.getenv("TARGET_COVERAGE", "0.68"))
|
| 38 |
+
W_KEPATUHAN = float(os.getenv("W_KEPATUHAN", "0.30"))
|
| 39 |
+
W_KINERJA = float(os.getenv("W_KINERJA", "0.70"))
|
| 40 |
|
| 41 |
+
USE_LLM = True
|
| 42 |
+
LLM_MODEL_NAME = os.getenv("LLM_MODEL_NAME", "meta-llama/Meta-Llama-3-8B-Instruct")
|
| 43 |
+
HF_TOKEN = (
|
| 44 |
+
os.getenv("HF_SECRET")
|
| 45 |
+
or os.getenv("HF_TOKEN")
|
| 46 |
+
or os.getenv("HUGGINGFACEHUB_API_TOKEN")
|
| 47 |
+
or os.getenv("HF_API_TOKEN")
|
| 48 |
+
)
|
| 49 |
|
| 50 |
# ============================================================
|
| 51 |
# 2) UTIL
|
| 52 |
# ============================================================
|
| 53 |
|
| 54 |
+
def _mtime(path_str: str):
|
| 55 |
+
p = Path(path_str)
|
| 56 |
+
return p.stat().st_mtime if p.exists() else None
|
| 57 |
+
|
| 58 |
def _canon(s: str) -> str:
|
| 59 |
return re.sub(r"[^a-z0-9]+", "", str(s).lower())
|
| 60 |
|
|
|
|
| 65 |
return " ".join(t.split())
|
| 66 |
|
| 67 |
def pick_col(df, candidates):
|
| 68 |
+
# exact
|
| 69 |
for c in candidates:
|
| 70 |
if c in df.columns:
|
| 71 |
return c
|
| 72 |
+
# canon
|
| 73 |
can_map = {_canon(c): c for c in df.columns}
|
| 74 |
for c in candidates:
|
| 75 |
k = _canon(c)
|
|
|
|
| 144 |
return float(num) / float(den)
|
| 145 |
|
| 146 |
def cap_bobot(cov: float) -> float:
|
| 147 |
+
# bobot normal: <68% -> cov/0.68, >=68% -> 1
|
| 148 |
if cov is None or pd.isna(cov) or cov <= 0:
|
| 149 |
+
return np.nan
|
| 150 |
return float(min(cov / TARGET_COVERAGE, 1.0))
|
| 151 |
|
| 152 |
+
def safe_round2(x):
|
| 153 |
+
try:
|
| 154 |
+
return round(float(x), 2)
|
| 155 |
+
except Exception:
|
| 156 |
+
return 0.0
|
| 157 |
+
|
| 158 |
def penalized_mean(row, cols):
|
| 159 |
vals = []
|
| 160 |
for c in cols:
|
|
|
|
| 166 |
vals.append(float(v))
|
| 167 |
return float(np.mean(vals)) if vals else 0.0
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
|
| 170 |
# ============================================================
|
| 171 |
# 3) INDIKATOR IPLM
|
|
|
|
| 222 |
}
|
| 223 |
alias_map = {_canon(k): v for k, v in alias_map_raw.items()}
|
| 224 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
|
| 226 |
# ============================================================
|
| 227 |
+
# 4) PIPELINE NASIONAL: YJ + MINMAX + SUBDIM/DIM/REAL
|
| 228 |
# ============================================================
|
| 229 |
|
| 230 |
def prepare_global(df_src: pd.DataFrame) -> pd.DataFrame:
|
|
|
|
| 232 |
return df_src
|
| 233 |
df = df_src.copy()
|
| 234 |
|
| 235 |
+
# rename indikator ke baku
|
| 236 |
rename_map = {}
|
| 237 |
for col in df.columns:
|
| 238 |
c = _canon(col)
|
|
|
|
| 247 |
df = df.rename(columns=rename_map)
|
| 248 |
|
| 249 |
available = [c for c in all_indicators if c in df.columns]
|
| 250 |
+
|
| 251 |
for c in available:
|
| 252 |
df[c] = df[c].apply(coerce_num)
|
| 253 |
|
| 254 |
+
# YJ + minmax nasional
|
| 255 |
for c in available:
|
| 256 |
x = df[c].astype(float).values
|
| 257 |
mask = ~np.isnan(x)
|
|
|
|
| 263 |
transformed[mask] = x[mask]
|
| 264 |
df[f"norm_{c}"] = minmax_norm(pd.Series(transformed, index=df.index))
|
| 265 |
|
| 266 |
+
df["sub_koleksi"] = df.apply(lambda r: penalized_mean(r, [c for c in koleksi_cols if c in available]), axis=1)
|
| 267 |
+
df["sub_sdm"] = df.apply(lambda r: penalized_mean(r, [c for c in sdm_cols if c in available]), axis=1)
|
| 268 |
+
df["sub_pelayanan"] = df.apply(lambda r: penalized_mean(r, [c for c in pelayanan_cols if c in available]), axis=1)
|
| 269 |
df["sub_pengelolaan"] = df.apply(lambda r: penalized_mean(r, [c for c in pengelolaan_cols if c in available]), axis=1)
|
| 270 |
|
| 271 |
df["dim_kepatuhan"] = df[["sub_koleksi","sub_sdm"]].mean(axis=1)
|
| 272 |
df["dim_kinerja"] = df[["sub_pelayanan","sub_pengelolaan"]].mean(axis=1)
|
| 273 |
|
| 274 |
df["Indeks_Real_0_100"] = 100 * (W_KEPATUHAN * df["dim_kepatuhan"] + W_KINERJA * df["dim_kinerja"])
|
| 275 |
+
|
| 276 |
for c in ["sub_koleksi","sub_sdm","sub_pelayanan","sub_pengelolaan","dim_kepatuhan","dim_kinerja","Indeks_Real_0_100"]:
|
| 277 |
df[c] = df[c].fillna(0.0)
|
| 278 |
|
| 279 |
return df
|
| 280 |
|
|
|
|
| 281 |
|
| 282 |
# ============================================================
|
| 283 |
+
# 5) CACHE LOADER (NO UPLOAD)
|
| 284 |
+
# ============================================================
|
| 285 |
+
|
| 286 |
+
_CACHE = {
|
| 287 |
+
"key": None,
|
| 288 |
+
"df_all": None,
|
| 289 |
+
"pop_kab": None,
|
| 290 |
+
"pop_prov": None,
|
| 291 |
+
"meta": None,
|
| 292 |
+
"info": None,
|
| 293 |
+
}
|
| 294 |
+
|
| 295 |
+
def load_default_files(force=False):
|
| 296 |
+
key = (DATA_FILE, POP_KAB, POP_PROV, _mtime(DATA_FILE), _mtime(POP_KAB), _mtime(POP_PROV))
|
| 297 |
+
|
| 298 |
+
if (not force) and _CACHE["key"] == key and _CACHE["df_all"] is not None:
|
| 299 |
+
return _CACHE["df_all"], _CACHE["pop_kab"], _CACHE["pop_prov"], _CACHE["meta"], _CACHE["info"]
|
| 300 |
+
|
| 301 |
+
# cek file
|
| 302 |
+
for p, label in [(DATA_FILE, "DM"), (POP_KAB, "POP_KAB"), (POP_PROV, "POP_PROV")]:
|
| 303 |
+
if not Path(p).exists():
|
| 304 |
+
info = f"β File {label} tidak ditemukan: `{p}`"
|
| 305 |
+
_CACHE.update({"key": key, "df_all": None, "pop_kab": None, "pop_prov": None, "meta": {}, "info": info})
|
| 306 |
+
return None, None, None, {}, info
|
| 307 |
+
|
| 308 |
+
# baca DM multi-sheet
|
| 309 |
+
fp = Path(DATA_FILE)
|
| 310 |
+
xls = pd.ExcelFile(fp)
|
| 311 |
+
frames = [pd.read_excel(fp, sheet_name=s) for s in xls.sheet_names]
|
| 312 |
+
df_raw = pd.concat(frames, ignore_index=True, sort=False)
|
| 313 |
+
|
| 314 |
+
prov_col = pick_col(df_raw, ["provinsi", "Provinsi", "PROVINSI"])
|
| 315 |
+
kab_col = pick_col(df_raw, ["kab_kota", "Kab/Kota", "Kab_Kota", "KAB/KOTA", "kabupaten_kota", "kota"])
|
| 316 |
+
kew_col = pick_col(df_raw, ["kewenangan", "jenis_kewenangan", "Kewenangan", "KEWENANGAN"])
|
| 317 |
+
jenis_col = pick_col(df_raw, ["jenis_perpustakaan", "Jenis Perpustakaan", "JENIS_PERPUSTAKAAN"])
|
| 318 |
+
nama_col = pick_col(df_raw, ["nm_perpustakaan","nama_perpustakaan", "Nama Perpustakaan", "nm_instansi_lembaga","nm_perpus"])
|
| 319 |
+
|
| 320 |
+
missing = []
|
| 321 |
+
if prov_col is None: missing.append("Provinsi")
|
| 322 |
+
if kab_col is None: missing.append("Kab/Kota")
|
| 323 |
+
if kew_col is None: missing.append("Kewenangan")
|
| 324 |
+
if jenis_col is None: missing.append("Jenis Perpustakaan")
|
| 325 |
+
if missing:
|
| 326 |
+
info = f"β Kolom wajib tidak ditemukan di DM: {', '.join(missing)}"
|
| 327 |
+
_CACHE.update({"key": key, "df_all": None, "pop_kab": None, "pop_prov": None, "meta": {}, "info": info})
|
| 328 |
+
return None, None, None, {}, info
|
| 329 |
+
|
| 330 |
+
# normalisasi jenis
|
| 331 |
+
val_map_jenis = {
|
| 332 |
+
"PERPUSTAKAAN SEKOLAH": "sekolah", "SEKOLAH": "sekolah",
|
| 333 |
+
"PERPUSTAKAAN UMUM": "umum", "UMUM": "umum", "PERPUSTAKAAN DAERAH": "umum",
|
| 334 |
+
"PERPUSTAKAAN KHUSUS": "khusus", "KHUSUS": "khusus",
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
+
df_raw["KEW_NORM"] = df_raw[kew_col].apply(norm_kew)
|
| 338 |
+
df_raw["_dataset"] = df_raw[jenis_col].astype(str).str.strip().str.upper().map(val_map_jenis)
|
| 339 |
+
df_raw["PROV_DISP"] = df_raw[prov_col].apply(_disp_text)
|
| 340 |
+
df_raw["KAB_DISP"] = df_raw[kab_col].apply(_disp_text)
|
| 341 |
+
|
| 342 |
+
# DEDUP
|
| 343 |
+
if nama_col and nama_col in df_raw.columns:
|
| 344 |
+
kcols = [prov_col, kab_col, kew_col, jenis_col, nama_col]
|
| 345 |
+
else:
|
| 346 |
+
kcols = [prov_col, kab_col, kew_col, jenis_col]
|
| 347 |
+
|
| 348 |
+
tmp = df_raw[kcols].astype(str).fillna("").apply(lambda s: s.str.strip(), axis=0)
|
| 349 |
+
df_raw["_row_key"] = tmp.apply(lambda r: "||".join(r.values.tolist()), axis=1).apply(_canon)
|
| 350 |
+
before = len(df_raw)
|
| 351 |
+
df_raw = df_raw.drop_duplicates(subset=["_row_key"], keep="first").copy()
|
| 352 |
+
after = len(df_raw)
|
| 353 |
+
|
| 354 |
+
# POP KAB
|
| 355 |
+
pk = pd.read_excel(POP_KAB)
|
| 356 |
+
c_prov = pick_col(pk, ["PROVINSI","Provinsi"])
|
| 357 |
+
c_kab = pick_col(pk, ["KABUPATEN_KOTA","Kab/Kota","Kabupaten/Kota","KAB/KOTA","Kabupaten_Kota"])
|
| 358 |
+
c_pop_umum = pick_col(pk, ["Pop_Umum","pop_umum","jumlah_populasi_umum","POP_UMUM"])
|
| 359 |
+
c_pop_sekolah = pick_col(pk, ["Pop_Sekolah","pop_sekolah","jumlah_populasi_sekolah","POP_SEKOLAH"])
|
| 360 |
+
|
| 361 |
+
if c_kab is None:
|
| 362 |
+
info = "β Populasi Kab/Kota: kolom Kab/Kota tidak ditemukan."
|
| 363 |
+
_CACHE.update({"key": key, "df_all": None, "pop_kab": None, "pop_prov": None, "meta": {}, "info": info})
|
| 364 |
+
return None, None, None, {}, info
|
| 365 |
+
|
| 366 |
+
pop_kab = pd.DataFrame({
|
| 367 |
+
"Provinsi_Label": pk[c_prov].astype(str).str.strip() if c_prov else "",
|
| 368 |
+
"Kab_Kota_Label": pk[c_kab].astype(str).str.strip(),
|
| 369 |
+
"Pop_Umum": pk[c_pop_umum].apply(coerce_num) if c_pop_umum else np.nan,
|
| 370 |
+
"Pop_Sekolah": pk[c_pop_sekolah].apply(coerce_num) if c_pop_sekolah else np.nan,
|
| 371 |
+
})
|
| 372 |
+
pop_kab["kab_key"] = pop_kab["Kab_Kota_Label"].apply(norm_kab_label)
|
| 373 |
+
pop_kab = pop_kab.groupby("kab_key", as_index=False).agg({
|
| 374 |
+
"Kab_Kota_Label":"first",
|
| 375 |
+
"Provinsi_Label":"first",
|
| 376 |
+
"Pop_Umum":"max",
|
| 377 |
+
"Pop_Sekolah":"max",
|
| 378 |
+
})
|
| 379 |
+
|
| 380 |
+
# POP PROV
|
| 381 |
+
pp = pd.read_excel(POP_PROV)
|
| 382 |
+
c_pr = pick_col(pp, ["Provinsi","PROVINSI","provinsi"])
|
| 383 |
+
c_total = pick_col(pp, ["total_pend","TOTAL_PEND","Pop_Sekolah_Prov","pop_sekolah_prov","sma","SMA","TOTAL_SMA","total_sma"])
|
| 384 |
+
if c_pr is None or c_total is None:
|
| 385 |
+
info = "β Populasi Provinsi: kolom Provinsi / total populasi sekolah tidak ditemukan."
|
| 386 |
+
_CACHE.update({"key": key, "df_all": None, "pop_kab": None, "pop_prov": None, "meta": {}, "info": info})
|
| 387 |
+
return None, None, None, {}, info
|
| 388 |
+
|
| 389 |
+
pop_prov = pd.DataFrame({
|
| 390 |
+
"Provinsi_Label": pp[c_pr].astype(str).str.strip(),
|
| 391 |
+
"Pop_Sekolah_Prov": pp[c_total].apply(coerce_num),
|
| 392 |
+
})
|
| 393 |
+
pop_prov["prov_key"] = pop_prov["Provinsi_Label"].apply(norm_prov_label)
|
| 394 |
+
pop_prov = pop_prov.groupby("prov_key", as_index=False).agg({
|
| 395 |
+
"Provinsi_Label":"first",
|
| 396 |
+
"Pop_Sekolah_Prov":"sum",
|
| 397 |
+
})
|
| 398 |
+
|
| 399 |
+
# PIPELINE NASIONAL (sekali)
|
| 400 |
+
df_all = prepare_global(df_raw)
|
| 401 |
+
|
| 402 |
+
meta = dict(
|
| 403 |
+
prov_col=prov_col, kab_col=kab_col, kew_col=kew_col, jenis_col=jenis_col, nama_col=nama_col
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
info = (
|
| 407 |
+
f"β
Mode NO UPLOAD (cache aktif)<br>"
|
| 408 |
+
f"β
DM: <b>{fp.name}</b> | Baris: {before} β dedup: {after}<br>"
|
| 409 |
+
f"β
Pop Kab/Kota: <b>{Path(POP_KAB).name}</b> (n={len(pop_kab)})<br>"
|
| 410 |
+
f"β
Pop Provinsi: <b>{Path(POP_PROV).name}</b> (n={len(pop_prov)})<br>"
|
| 411 |
+
f"π mtime: DM={time.ctime(_mtime(DATA_FILE))} | Kab={time.ctime(_mtime(POP_KAB))} | Prov={time.ctime(_mtime(POP_PROV))}"
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
_CACHE.update({"key": key, "df_all": df_all, "pop_kab": pop_kab, "pop_prov": pop_prov, "meta": meta, "info": info})
|
| 415 |
+
return df_all, pop_kab, pop_prov, meta, info
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
# ============================================================
|
| 419 |
+
# 6) PENALTI 68% -> FINAL + VERIF (NO DECIMALS)
|
| 420 |
# ============================================================
|
| 421 |
|
| 422 |
+
def apply_penalty_68(df_filtered: pd.DataFrame, pop_kab: pd.DataFrame, pop_prov: pd.DataFrame, kew_value: str):
|
| 423 |
if df_filtered is None or df_filtered.empty:
|
| 424 |
return df_filtered, pd.DataFrame()
|
| 425 |
|
|
|
|
| 428 |
|
| 429 |
df["bobot_coverage"] = 1.0
|
| 430 |
df["coverage"] = np.nan
|
| 431 |
+
verif_df = pd.DataFrame()
|
| 432 |
+
|
| 433 |
+
def _bobot_or_one(b):
|
| 434 |
+
if b is None or pd.isna(b):
|
| 435 |
+
return 1.0
|
| 436 |
+
return float(b)
|
| 437 |
|
| 438 |
+
# --- KAB/KOTA ---
|
| 439 |
+
if ("KAB" in kew_norm or "KOTA" in kew_norm) and pop_kab is not None and not pop_kab.empty:
|
| 440 |
tmp = df.copy()
|
| 441 |
tmp["kab_key"] = tmp["KAB_DISP"].apply(norm_kab_label)
|
| 442 |
|
| 443 |
g = tmp.groupby(["kab_key","_dataset"]).size().rename("n_sampel").reset_index()
|
| 444 |
g_piv = g.pivot(index="kab_key", columns="_dataset", values="n_sampel").fillna(0)
|
| 445 |
|
| 446 |
+
pop = pop_kab.set_index("kab_key")
|
| 447 |
|
| 448 |
rows = []
|
| 449 |
for kk in g_piv.index:
|
|
|
|
| 478 |
})
|
| 479 |
|
| 480 |
verif_df = pd.DataFrame(rows)
|
| 481 |
+
verif_df["Catatan"] = ""
|
| 482 |
+
verif_df.loc[verif_df["Pop_Sekolah"].isna() | (verif_df["Pop_Sekolah"] <= 0), "Catatan"] += "Pop_Sekolah_tidak_valid; "
|
| 483 |
+
verif_df.loc[verif_df["Pop_Umum"].isna() | (verif_df["Pop_Umum"] <= 0), "Catatan"] += "Pop_Umum_tidak_valid; "
|
| 484 |
|
|
|
|
| 485 |
int_cols = ["Pop_Sekolah","Sampel_Sekolah","GAP_Ke_68_Sekolah","Pop_Umum","Sampel_Umum","GAP_Ke_68_Umum"]
|
| 486 |
pct_cols = ["Coverage_Sekolah_%","Bobot_Sekolah_68_%","Coverage_Umum_%","Bobot_Umum_68_%"]
|
| 487 |
for c in int_cols:
|
|
|
|
| 491 |
if c in verif_df.columns:
|
| 492 |
verif_df[c] = verif_df[c].fillna(0).round(0).astype(int)
|
| 493 |
|
| 494 |
+
bobot_map_sek = {norm_kab_label(r["Kab/Kota"]): _bobot_or_one(float(r["Bobot_Sekolah_68_%"]) / 100.0) for _, r in verif_df.iterrows()}
|
| 495 |
+
bobot_map_um = {norm_kab_label(r["Kab/Kota"]): _bobot_or_one(float(r["Bobot_Umum_68_%"]) / 100.0) for _, r in verif_df.iterrows()}
|
| 496 |
|
| 497 |
+
cov_map_sek = {norm_kab_label(r["Kab/Kota"]): (float(r["Coverage_Sekolah_%"]) / 100.0) for _, r in verif_df.iterrows()}
|
| 498 |
+
cov_map_um = {norm_kab_label(r["Kab/Kota"]): (float(r["Coverage_Umum_%"]) / 100.0) for _, r in verif_df.iterrows()}
|
| 499 |
|
| 500 |
df["kab_key"] = df["KAB_DISP"].apply(norm_kab_label)
|
| 501 |
|
|
|
|
| 505 |
if ds == "khusus":
|
| 506 |
return 1.0
|
| 507 |
if ds == "sekolah":
|
| 508 |
+
return float(bobot_map_sek.get(kk, 1.0))
|
| 509 |
if ds == "umum":
|
| 510 |
+
return float(bobot_map_um.get(kk, 1.0))
|
| 511 |
return 1.0
|
| 512 |
|
| 513 |
def row_cov(r):
|
|
|
|
| 522 |
df["bobot_coverage"] = df.apply(row_weight, axis=1)
|
| 523 |
df["coverage"] = df.apply(row_cov, axis=1)
|
| 524 |
|
| 525 |
+
# --- PROVINSI ---
|
| 526 |
+
elif ("PROV" in kew_norm) and pop_prov is not None and not pop_prov.empty:
|
| 527 |
tmp = df.copy()
|
| 528 |
tmp["prov_key"] = tmp["PROV_DISP"].apply(norm_prov_label)
|
| 529 |
|
| 530 |
g = tmp.groupby(["prov_key","_dataset"]).size().rename("n_sampel").reset_index()
|
| 531 |
g_piv = g.pivot(index="prov_key", columns="_dataset", values="n_sampel").fillna(0)
|
| 532 |
+
pop = pop_prov.set_index("prov_key")
|
| 533 |
|
| 534 |
rows = []
|
| 535 |
for pk in g_piv.index:
|
| 536 |
pop_sek = pop.loc[pk, "Pop_Sekolah_Prov"] if pk in pop.index else np.nan
|
| 537 |
n_sek = float(g_piv.loc[pk].get("sekolah", 0))
|
| 538 |
+
|
| 539 |
cov_sek = safe_div(n_sek, pop_sek)
|
| 540 |
bobot_sek = cap_bobot(cov_sek)
|
| 541 |
+
|
| 542 |
target_sek = (TARGET_COVERAGE * pop_sek) if not pd.isna(pop_sek) else np.nan
|
| 543 |
|
| 544 |
rows.append({
|
|
|
|
| 551 |
})
|
| 552 |
|
| 553 |
verif_df = pd.DataFrame(rows)
|
| 554 |
+
verif_df["Catatan"] = ""
|
| 555 |
+
verif_df.loc[verif_df["Pop_Sekolah"].isna() | (verif_df["Pop_Sekolah"] <= 0), "Catatan"] += "Pop_Sekolah_tidak_valid; "
|
| 556 |
|
| 557 |
int_cols = ["Pop_Sekolah","Sampel_Sekolah","GAP_Ke_68_Sekolah"]
|
| 558 |
pct_cols = ["Coverage_Sekolah_%","Bobot_Sekolah_68_%"]
|
|
|
|
| 563 |
if c in verif_df.columns:
|
| 564 |
verif_df[c] = verif_df[c].fillna(0).round(0).astype(int)
|
| 565 |
|
| 566 |
+
bobot_map = {norm_prov_label(r["Provinsi"]): _bobot_or_one(float(r["Bobot_Sekolah_68_%"]) / 100.0) for _, r in verif_df.iterrows()}
|
| 567 |
+
cov_map = {norm_prov_label(r["Provinsi"]): (float(r["Coverage_Sekolah_%"]) / 100.0) for _, r in verif_df.iterrows()}
|
| 568 |
|
| 569 |
df["prov_key"] = df["PROV_DISP"].apply(norm_prov_label)
|
| 570 |
|
|
|
|
| 573 |
if ds == "khusus":
|
| 574 |
return 1.0
|
| 575 |
if ds == "sekolah":
|
| 576 |
+
return float(bobot_map.get(r.get("prov_key", None), 1.0))
|
| 577 |
return 1.0
|
| 578 |
|
| 579 |
def row_cov(r):
|
|
|
|
| 584 |
df["bobot_coverage"] = df.apply(row_weight, axis=1)
|
| 585 |
df["coverage"] = df.apply(row_cov, axis=1)
|
| 586 |
|
| 587 |
+
# FINAL
|
| 588 |
+
df["Indeks_Final_0_100"] = (df["Indeks_Real_0_100"].fillna(0.0) * df["bobot_coverage"].fillna(1.0)).fillna(0.0)
|
|
|
|
|
|
|
| 589 |
return df, verif_df
|
| 590 |
|
| 591 |
+
|
| 592 |
+
# ============================================================
|
| 593 |
+
# 7) VIEW: DETAIL + AGREGAT
|
| 594 |
+
# ============================================================
|
| 595 |
+
|
| 596 |
+
def build_views(df: pd.DataFrame, meta: dict):
|
| 597 |
+
if df is None or df.empty:
|
| 598 |
+
return pd.DataFrame()
|
| 599 |
+
|
| 600 |
+
base_cols = ["PROV_DISP", "KAB_DISP", "KEW_NORM", "_dataset"]
|
| 601 |
+
if meta.get("nama_col") and meta["nama_col"] in df.columns:
|
| 602 |
+
df = df.copy()
|
| 603 |
+
df["nm_perpustakaan"] = df[meta["nama_col"]].astype(str)
|
| 604 |
+
base_cols.insert(2, "nm_perpustakaan")
|
| 605 |
+
|
| 606 |
+
keep = base_cols + [
|
| 607 |
+
"sub_koleksi","sub_sdm","sub_pelayanan","sub_pengelolaan",
|
| 608 |
+
"dim_kepatuhan","dim_kinerja",
|
| 609 |
+
"Indeks_Final_0_100"
|
| 610 |
+
]
|
| 611 |
+
keep = [c for c in keep if c in df.columns]
|
| 612 |
+
out = df[keep].copy()
|
| 613 |
+
out = out.rename(columns={"PROV_DISP":"Provinsi","KAB_DISP":"Kab/Kota","_dataset":"Jenis"})
|
| 614 |
+
|
| 615 |
+
out["Indeks_Final_0_100"] = out["Indeks_Final_0_100"].apply(safe_round2)
|
| 616 |
+
for c in ["sub_koleksi","sub_sdm","sub_pelayanan","sub_pengelolaan","dim_kepatuhan","dim_kinerja"]:
|
| 617 |
+
if c in out.columns:
|
| 618 |
+
out[c] = out[c].apply(lambda x: round(float(x), 3) if pd.notna(x) else 0.0)
|
| 619 |
+
|
| 620 |
+
return out
|
| 621 |
+
|
| 622 |
+
def build_aggregate(df_view: pd.DataFrame):
|
| 623 |
+
if df_view is None or df_view.empty:
|
| 624 |
+
return pd.DataFrame()
|
| 625 |
+
|
| 626 |
+
grp = df_view.groupby("Jenis", dropna=False).agg(
|
| 627 |
+
Jumlah=("Jenis","size"),
|
| 628 |
+
Rata2_sub_koleksi=("sub_koleksi","mean"),
|
| 629 |
+
Rata2_sub_sdm=("sub_sdm","mean"),
|
| 630 |
+
Rata2_sub_pelayanan=("sub_pelayanan","mean"),
|
| 631 |
+
Rata2_sub_pengelolaan=("sub_pengelolaan","mean"),
|
| 632 |
+
Rata2_dim_kepatuhan=("dim_kepatuhan","mean"),
|
| 633 |
+
Rata2_dim_kinerja=("dim_kinerja","mean"),
|
| 634 |
+
Rata2_Indeks_Final_0_100=("Indeks_Final_0_100","mean"),
|
| 635 |
+
).reset_index()
|
| 636 |
+
|
| 637 |
+
for c in grp.columns:
|
| 638 |
+
if c.startswith("Rata2_"):
|
| 639 |
+
grp[c] = grp[c].apply(lambda x: round(float(x), 3) if pd.notna(x) else 0.0)
|
| 640 |
+
|
| 641 |
+
overall = {
|
| 642 |
+
"Jenis":"Rata-rata keseluruhan",
|
| 643 |
+
"Jumlah": int(df_view.shape[0]),
|
| 644 |
+
"Rata2_sub_koleksi": float(df_view["sub_koleksi"].mean()),
|
| 645 |
+
"Rata2_sub_sdm": float(df_view["sub_sdm"].mean()),
|
| 646 |
+
"Rata2_sub_pelayanan": float(df_view["sub_pelayanan"].mean()),
|
| 647 |
+
"Rata2_sub_pengelolaan": float(df_view["sub_pengelolaan"].mean()),
|
| 648 |
+
"Rata2_dim_kepatuhan": float(df_view["dim_kepatuhan"].mean()),
|
| 649 |
+
"Rata2_dim_kinerja": float(df_view["dim_kinerja"].mean()),
|
| 650 |
+
"Rata2_Indeks_Final_0_100": float(df_view["Indeks_Final_0_100"].mean()),
|
| 651 |
+
}
|
| 652 |
+
grp = pd.concat([grp, pd.DataFrame([overall])], ignore_index=True)
|
| 653 |
+
|
| 654 |
+
for c in grp.columns:
|
| 655 |
+
if c.startswith("Rata2_"):
|
| 656 |
+
grp[c] = grp[c].apply(lambda x: round(float(x), 3) if pd.notna(x) else 0.0)
|
| 657 |
+
|
| 658 |
+
return grp
|
| 659 |
+
|
| 660 |
+
|
| 661 |
# ============================================================
|
| 662 |
+
# 8) BELL CURVE
|
| 663 |
# ============================================================
|
| 664 |
|
| 665 |
+
def make_bell_figure(df_all: pd.DataFrame, title: str, index_col: str, name_col: str = None, min_points: int = 5) -> go.Figure:
|
| 666 |
fig = go.Figure()
|
| 667 |
+
fig.update_layout(title=title, xaxis_title="Indeks (0β100)", yaxis_title="Kepadatan (relatif)")
|
| 668 |
+
|
| 669 |
+
if df_all is None or df_all.empty or index_col not in df_all.columns:
|
| 670 |
return fig
|
| 671 |
|
| 672 |
+
dfp = df_all.dropna(subset=[index_col]).copy()
|
| 673 |
if dfp.empty or len(dfp) < min_points:
|
| 674 |
+
fig.add_annotation(
|
| 675 |
+
text="Grafik tidak ditampilkan (data terlalu sedikit).",
|
| 676 |
+
x=0.5, y=0.5, xref="paper", yref="paper", showarrow=False
|
|
|
|
|
|
|
|
|
|
|
|
|
| 677 |
)
|
| 678 |
return fig
|
| 679 |
|
| 680 |
+
x = dfp[index_col].astype(float).values
|
| 681 |
+
mu = float(np.mean(x))
|
| 682 |
+
sigma = float(np.std(x, ddof=1)) if len(x) > 1 else 1.0
|
| 683 |
+
sigma = max(sigma, 1e-6)
|
|
|
|
| 684 |
|
| 685 |
+
xs = np.linspace(max(0, np.min(x) - 5), min(100, np.max(x) + 5), 200)
|
| 686 |
pdf = (1.0 / (sigma * np.sqrt(2 * np.pi))) * np.exp(-0.5 * ((xs - mu) / sigma) ** 2)
|
| 687 |
+
pdf = pdf / max(pdf.max(), 1e-9)
|
| 688 |
|
| 689 |
if name_col and name_col in dfp.columns:
|
| 690 |
+
hover = [f"{str(n)}<br>Indeks: {v:.2f}" for n, v in zip(dfp[name_col], x)]
|
| 691 |
else:
|
| 692 |
+
hover = [f"Indeks: {v:.2f}" for v in x]
|
| 693 |
|
| 694 |
fig.add_trace(go.Scatter(x=xs, y=pdf, mode="lines", name="Bell curve", hoverinfo="skip"))
|
| 695 |
+
fig.add_trace(go.Scatter(x=x, y=np.zeros_like(x), mode="markers", name="Perpustakaan",
|
| 696 |
+
hovertext=hover, hovertemplate="%{hovertext}<extra></extra>"))
|
|
|
|
|
|
|
|
|
|
| 697 |
|
| 698 |
+
q1, q2, q3 = np.quantile(x, [0.25, 0.5, 0.75])
|
| 699 |
for q, label in [(q1, "Q1"), (q2, "Q2 (Median)"), (q3, "Q3")]:
|
| 700 |
fig.add_trace(go.Scatter(
|
| 701 |
x=[q, q], y=[0, 1.05],
|
|
|
|
| 704 |
))
|
| 705 |
|
| 706 |
fig.update_layout(
|
| 707 |
+
xaxis_title="Indeks FINAL IPLM (0β100)",
|
| 708 |
+
yaxis=dict(showticklabels=False, range=[0, 1.2]),
|
|
|
|
|
|
|
| 709 |
margin=dict(l=40, r=20, t=60, b=40),
|
| 710 |
hovermode="x"
|
| 711 |
)
|
| 712 |
return fig
|
| 713 |
|
| 714 |
+
|
| 715 |
# ============================================================
|
| 716 |
+
# 9) LLM
|
|
|
|
| 717 |
# ============================================================
|
| 718 |
|
| 719 |
+
_HF_CLIENT = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 720 |
|
| 721 |
+
def get_llm_client():
|
| 722 |
+
global _HF_CLIENT
|
| 723 |
+
if _HF_CLIENT is not None:
|
| 724 |
+
return _HF_CLIENT
|
| 725 |
+
try:
|
| 726 |
+
_HF_CLIENT = InferenceClient(model=LLM_MODEL_NAME, token=HF_TOKEN) if HF_TOKEN else InferenceClient(model=LLM_MODEL_NAME)
|
| 727 |
+
return _HF_CLIENT
|
| 728 |
+
except Exception:
|
| 729 |
+
_HF_CLIENT = None
|
| 730 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 731 |
|
| 732 |
+
def build_context(detail_df: pd.DataFrame, agg_df: pd.DataFrame, verif_df: pd.DataFrame, wilayah: str, kew: str) -> str:
|
| 733 |
lines = []
|
| 734 |
lines.append(f"Wilayah: {wilayah}")
|
| 735 |
+
lines.append(f"Kewenangan: {kew}")
|
| 736 |
+
lines.append(f"Jumlah perpustakaan sampel: {len(detail_df)}")
|
| 737 |
+
if "Indeks_Final_0_100" in detail_df.columns:
|
| 738 |
+
lines.append(f"Rata-rata Indeks FINAL: {detail_df['Indeks_Final_0_100'].mean(skipna=True):.2f}")
|
| 739 |
+
for col in ["sub_koleksi","sub_sdm","sub_pelayanan","sub_pengelolaan","dim_kepatuhan","dim_kinerja"]:
|
| 740 |
+
if col in detail_df.columns:
|
| 741 |
+
lines.append(f"Rata-rata {col}: {detail_df[col].mean(skipna=True):.3f}")
|
| 742 |
|
| 743 |
+
if agg_df is not None and not agg_df.empty:
|
| 744 |
+
lines.append("\nRingkasan per jenis:")
|
| 745 |
+
for _, r in agg_df.iterrows():
|
| 746 |
+
jenis = r.get("Jenis", "")
|
| 747 |
+
if jenis == "Rata-rata keseluruhan":
|
| 748 |
+
continue
|
| 749 |
+
lines.append(f"- {jenis}: n={int(r['Jumlah'])}, Indeks_FINAL={float(r['Rata2_Indeks_Final_0_100']):.2f}")
|
| 750 |
|
| 751 |
+
if verif_df is not None and not verif_df.empty:
|
| 752 |
+
lines.append("\nCatatan verifikasi coverage 68% (ringkas):")
|
| 753 |
+
gap_cols = [c for c in verif_df.columns if c.startswith("GAP_Ke_68")]
|
| 754 |
+
if gap_cols:
|
| 755 |
+
tmp = verif_df.copy()
|
| 756 |
+
tmp["GAP_MAX"] = tmp[gap_cols].max(axis=1)
|
| 757 |
+
tmp = tmp.sort_values("GAP_MAX", ascending=False).head(5)
|
| 758 |
+
for _, r in tmp.iterrows():
|
| 759 |
+
name = r.get("Kab/Kota", r.get("Provinsi",""))
|
| 760 |
+
lines.append(f"- {name}: GAP maks={int(r['GAP_MAX'])}")
|
| 761 |
+
|
| 762 |
+
if "Catatan" in verif_df.columns:
|
| 763 |
+
n_bad = (verif_df["Catatan"].astype(str).str.contains("tidak_valid", na=False)).sum()
|
| 764 |
+
if n_bad > 0:
|
| 765 |
+
lines.append(f"\nCatatan data: ada {int(n_bad)} wilayah dengan populasi tidak valid β bobot diset 1 (tanpa penalti).")
|
| 766 |
+
|
| 767 |
+
return "\n".join(lines)
|
| 768 |
+
|
| 769 |
+
def generate_llm_analysis(detail_df: pd.DataFrame, agg_df: pd.DataFrame, verif_df: pd.DataFrame, wilayah: str, kew: str) -> str:
|
| 770 |
+
ctx = build_context(detail_df, agg_df, verif_df, wilayah, kew)
|
| 771 |
client = get_llm_client()
|
| 772 |
if client is None or not USE_LLM:
|
| 773 |
+
return "Analisis otomatis (LLM) tidak tersedia. Pastikan token HuggingFace tersedia dan model bisa diakses."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 774 |
|
| 775 |
system_prompt = (
|
| 776 |
+
"Anda adalah analis kebijakan perpustakaan dan literasi di Indonesia. "
|
| 777 |
+
"Tugas Anda menyusun analisis berbasis data IPLM secara formal, tajam, dan operasional."
|
|
|
|
| 778 |
)
|
|
|
|
| 779 |
user_prompt = f"""
|
| 780 |
+
DATA RINGKAS IPLM (SETELAH PENALTI COVERAGE 68%):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 781 |
|
| 782 |
+
{ctx}
|
|
|
|
| 783 |
|
| 784 |
+
TULISKAN ANALISIS BAHASA INDONESIA FORMAL, STRUKTUR:
|
| 785 |
+
1) Gambaran umum kondisi wilayah (1 paragraf).
|
| 786 |
+
2) Analisis capaian subdimensi & dimensi (2 paragraf). Jelaskan arti angka secara substantif.
|
| 787 |
+
3) Analisis risiko/kesenjangan coverage 68% dan implikasinya (1 paragraf).
|
| 788 |
+
4) Rekomendasi program 3β5 tahun (2 paragraf naratif). Harus konkret dan bisa dieksekusi.
|
| 789 |
|
| 790 |
+
ATURAN:
|
| 791 |
+
- Jangan pakai label menilai eksplisit seperti "rendah/sedang/tinggi".
|
| 792 |
+
- Gunakan frasa netral: "masih memiliki ruang penguatan", "memerlukan konsolidasi", dst.
|
| 793 |
+
- Fokus pada Indeks FINAL (setelah penalti 68%).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 794 |
"""
|
|
|
|
| 795 |
try:
|
| 796 |
resp = client.chat_completion(
|
| 797 |
model=LLM_MODEL_NAME,
|
| 798 |
+
messages=[{"role":"system","content":system_prompt},{"role":"user","content":user_prompt}],
|
| 799 |
+
max_tokens=1100,
|
|
|
|
|
|
|
|
|
|
| 800 |
temperature=0.25,
|
| 801 |
top_p=0.9,
|
| 802 |
)
|
| 803 |
text = resp.choices[0].message.content.strip()
|
| 804 |
+
return text if text else "LLM mengembalikan respon kosong."
|
|
|
|
|
|
|
| 805 |
except Exception as e:
|
| 806 |
+
return f"β οΈ Error saat memanggil LLM: {repr(e)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 807 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 808 |
|
| 809 |
+
# ============================================================
|
| 810 |
+
# 10) WORD
|
| 811 |
+
# ============================================================
|
|
|
|
| 812 |
|
| 813 |
+
def generate_word_report(detail_df: pd.DataFrame, agg_df: pd.DataFrame, verif_df: pd.DataFrame,
|
| 814 |
+
wilayah: str, kew: str, analysis_text: str) -> str:
|
| 815 |
doc = Document()
|
| 816 |
+
doc.add_heading(f"Laporan IPLM (FINAL) β {wilayah}", level=1)
|
| 817 |
+
doc.add_paragraph(f"Kewenangan: {kew}")
|
| 818 |
+
doc.add_paragraph("Catatan: Indeks FINAL memperhitungkan penalti coverage 68% (perpustakaan khusus tidak dikenai penalti).")
|
| 819 |
+
doc.add_paragraph("Jika populasi wilayah tidak valid/tidak ditemukan, bobot coverage diset 1 (tanpa penalti) dan dicatat pada tabel verifikasi.")
|
|
|
|
| 820 |
|
| 821 |
+
doc.add_heading("Ringkasan Utama", level=2)
|
| 822 |
+
if detail_df is not None and not detail_df.empty and "Indeks_Final_0_100" in detail_df.columns:
|
| 823 |
+
doc.add_paragraph(f"Jumlah perpustakaan: {len(detail_df)}")
|
| 824 |
+
doc.add_paragraph(f"Rata-rata Indeks FINAL: {detail_df['Indeks_Final_0_100'].mean(skipna=True):.2f}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 825 |
|
| 826 |
+
doc.add_heading("Agregat (sub/dim + Indeks FINAL)", level=2)
|
| 827 |
if agg_df is not None and not agg_df.empty:
|
| 828 |
table = doc.add_table(rows=1, cols=len(agg_df.columns))
|
| 829 |
hdr = table.rows[0].cells
|
| 830 |
for i, c in enumerate(agg_df.columns):
|
| 831 |
hdr[i].text = str(c)
|
| 832 |
for _, row in agg_df.iterrows():
|
| 833 |
+
cells = table.add_row().cells
|
| 834 |
for i, c in enumerate(agg_df.columns):
|
| 835 |
+
cells[i].text = str(row[c])
|
| 836 |
else:
|
| 837 |
+
doc.add_paragraph("Agregat tidak tersedia.")
|
| 838 |
|
| 839 |
+
doc.add_heading("Verifikasi Coverage & GAP menuju 68%", level=2)
|
| 840 |
if verif_df is not None and not verif_df.empty:
|
| 841 |
+
table = doc.add_table(rows=1, cols=len(verif_df.columns))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 842 |
hdr = table.rows[0].cells
|
| 843 |
+
for i, c in enumerate(verif_df.columns):
|
| 844 |
hdr[i].text = str(c)
|
| 845 |
+
for _, row in verif_df.iterrows():
|
| 846 |
+
cells = table.add_row().cells
|
| 847 |
+
for i, c in enumerate(verif_df.columns):
|
| 848 |
+
cells[i].text = str(row[c])
|
| 849 |
else:
|
| 850 |
+
doc.add_paragraph("Tidak ada tabel verifikasi untuk filter ini.")
|
| 851 |
|
| 852 |
+
doc.add_heading("Analisis Naratif (LLM)", level=2)
|
| 853 |
+
for p in (analysis_text or "").split("\n"):
|
| 854 |
+
if p.strip():
|
| 855 |
+
doc.add_paragraph(p.strip())
|
| 856 |
|
| 857 |
outpath = tempfile.mktemp(suffix=".docx")
|
| 858 |
doc.save(outpath)
|
| 859 |
return outpath
|
| 860 |
|
| 861 |
+
|
| 862 |
# ============================================================
|
| 863 |
+
# 11) CORE RUN
|
| 864 |
# ============================================================
|
| 865 |
|
| 866 |
+
def _empty_outputs(msg="β οΈ Data belum siap."):
|
| 867 |
+
empty = pd.DataFrame()
|
| 868 |
+
empty_fig = go.Figure()
|
| 869 |
+
return (
|
| 870 |
+
empty, empty, empty,
|
| 871 |
+
None, None, None, None,
|
| 872 |
+
empty_fig, empty_fig, empty_fig, empty_fig,
|
| 873 |
+
msg, "Analisis belum tersedia."
|
| 874 |
+
)
|
| 875 |
+
|
| 876 |
+
def run_calc(prov_value, kab_value, kew_value, df_all, pop_kab, pop_prov, meta):
|
| 877 |
+
try:
|
| 878 |
+
if df_all is None or (isinstance(df_all, pd.DataFrame) and df_all.empty):
|
| 879 |
+
return _empty_outputs("β οΈ Data belum ter-load. Klik Reload Data.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 880 |
|
| 881 |
+
df = df_all.copy()
|
|
|
|
|
|
|
| 882 |
|
| 883 |
+
if prov_value and prov_value != "(Semua)":
|
| 884 |
+
df = df[df["PROV_DISP"] == prov_value]
|
| 885 |
+
if kab_value and kab_value != "(Semua)":
|
| 886 |
+
df = df[df["KAB_DISP"] == kab_value]
|
| 887 |
+
if kew_value and kew_value != "(Semua)":
|
| 888 |
+
df = df[df["KEW_NORM"] == kew_value]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 889 |
|
| 890 |
+
if df.empty:
|
| 891 |
+
return _empty_outputs("Tidak ada data untuk filter ini.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 892 |
|
| 893 |
+
df_pen, verif_df = apply_penalty_68(df, pop_kab, pop_prov, kew_value)
|
| 894 |
+
detail_view = build_views(df_pen, meta)
|
| 895 |
+
agg_view = build_aggregate(detail_view)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 896 |
|
| 897 |
+
name_col = "nm_perpustakaan" if "nm_perpustakaan" in detail_view.columns else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
|
| 899 |
+
fig_all = make_bell_figure(detail_view, "Bell Curve Indeks FINAL β Semua Perpustakaan", "Indeks_Final_0_100", name_col=name_col, min_points=5)
|
| 900 |
+
fig_sek = make_bell_figure(detail_view[detail_view["Jenis"]=="sekolah"], "Bell Curve Indeks FINAL β Perpustakaan Sekolah", "Indeks_Final_0_100", name_col=name_col, min_points=3)
|
| 901 |
+
fig_um = make_bell_figure(detail_view[detail_view["Jenis"]=="umum"], "Bell Curve Indeks FINAL β Perpustakaan Umum", "Indeks_Final_0_100", name_col=name_col, min_points=3)
|
| 902 |
+
fig_kh = make_bell_figure(detail_view[detail_view["Jenis"]=="khusus"], "Bell Curve Indeks FINAL β Perpustakaan Khusus", "Indeks_Final_0_100", name_col=name_col, min_points=3)
|
| 903 |
+
|
| 904 |
+
tmpdir = tempfile.mkdtemp()
|
| 905 |
+
prov_slug = (_canon(prov_value or "SEMUA").upper() or "SEMUA")
|
| 906 |
+
kab_slug = (_canon(kab_value or "SEMUA").upper() or "SEMUA")
|
| 907 |
+
kew_slug = (_canon(kew_value or "SEMUA").upper() or "SEMUA")
|
| 908 |
+
|
| 909 |
+
agg_path = str(Path(tmpdir) / f"IPLM_Agregat_RINGKAS_{prov_slug}_{kab_slug}_{kew_slug}.xlsx")
|
| 910 |
+
det_path = str(Path(tmpdir) / f"IPLM_Detail_RINGKAS_{prov_slug}_{kab_slug}_{kew_slug}.xlsx")
|
| 911 |
+
ver_path = str(Path(tmpdir) / f"IPLM_VerifikasiCoverage_{prov_slug}_{kab_slug}_{kew_slug}.xlsx")
|
| 912 |
+
|
| 913 |
+
agg_view.to_excel(agg_path, index=False)
|
| 914 |
+
detail_view.to_excel(det_path, index=False)
|
| 915 |
+
(verif_df if verif_df is not None else pd.DataFrame()).to_excel(ver_path, index=False)
|
| 916 |
+
|
| 917 |
+
wilayah = kab_value if (kab_value and kab_value != "(Semua)") else (prov_value if (prov_value and prov_value != "(Semua)") else "Nasional/All")
|
| 918 |
+
analysis_text = generate_llm_analysis(detail_view, agg_view, verif_df, wilayah, kew_value or "(Semua)")
|
| 919 |
+
word_path = generate_word_report(detail_view, agg_view, verif_df, wilayah, kew_value or "(Semua)", analysis_text)
|
| 920 |
+
|
| 921 |
+
msg = f"β
Berhasil dihitung: {len(detail_view)} perpustakaan | Output: Indeks FINAL (penalti 68%)"
|
| 922 |
return (
|
| 923 |
+
agg_view, detail_view, verif_df,
|
| 924 |
+
agg_path, det_path, ver_path, word_path,
|
| 925 |
+
fig_all, fig_sek, fig_um, fig_kh,
|
| 926 |
+
msg, analysis_text
|
|
|
|
|
|
|
| 927 |
)
|
| 928 |
+
except Exception as e:
|
| 929 |
+
return _empty_outputs(f"β οΈ Runtime error: {repr(e)}")
|
| 930 |
|
|
|
|
|
|
|
| 931 |
|
| 932 |
+
# ============================================================
|
| 933 |
+
# 12) UI (NO UPLOAD) β AUTO LOAD + RELOAD BUTTON
|
| 934 |
+
# ============================================================
|
| 935 |
|
| 936 |
+
def ui_load(force=False):
|
| 937 |
+
df_all, pop_kab, pop_prov, meta, info = load_default_files(force=force)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 938 |
|
| 939 |
+
if df_all is None or (isinstance(df_all, pd.DataFrame) and df_all.empty):
|
| 940 |
+
return (
|
| 941 |
+
None, None, None, {}, info,
|
| 942 |
+
gr.update(choices=["(Semua)"], value="(Semua)"),
|
| 943 |
+
gr.update(choices=["(Semua)"], value="(Semua)"),
|
| 944 |
+
gr.update(choices=["(Semua)"], value="(Semua)")
|
| 945 |
+
)
|
| 946 |
+
|
| 947 |
+
prov_choices = ["(Semua)"] + sorted([x for x in df_all["PROV_DISP"].dropna().unique().tolist() if x])
|
| 948 |
+
kab_choices = ["(Semua)"] + sorted([x for x in df_all["KAB_DISP"].dropna().unique().tolist() if x])
|
| 949 |
+
kew_choices = ["(Semua)"] + sorted([x for x in df_all["KEW_NORM"].dropna().unique().tolist() if x])
|
| 950 |
+
|
| 951 |
+
default_kew = "KAB/KOTA" if "KAB/KOTA" in kew_choices else "(Semua)"
|
| 952 |
|
|
|
|
| 953 |
return (
|
| 954 |
+
df_all, pop_kab, pop_prov, meta, info,
|
| 955 |
+
gr.update(choices=prov_choices, value="(Semua)"),
|
| 956 |
+
gr.update(choices=kab_choices, value="(Semua)"),
|
| 957 |
+
gr.update(choices=kew_choices, value=default_kew)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 958 |
)
|
| 959 |
|
| 960 |
+
def on_prov_change(prov_value):
|
| 961 |
+
# Aman: ambil dari cache loader langsung, bukan state_df (yang bisa None saat load)
|
| 962 |
+
df_all, _, _, _, _ = load_default_files(force=False)
|
| 963 |
+
if df_all is None or df_all.empty:
|
| 964 |
+
return gr.update(choices=["(Semua)"], value="(Semua)")
|
| 965 |
|
| 966 |
+
if prov_value is None or prov_value == "(Semua)":
|
| 967 |
+
vals = df_all["KAB_DISP"].dropna().unique().tolist()
|
| 968 |
+
else:
|
| 969 |
+
vals = df_all.loc[df_all["PROV_DISP"] == prov_value, "KAB_DISP"].dropna().unique().tolist()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 970 |
|
| 971 |
+
vals = sorted([v for v in vals if v])
|
| 972 |
+
return gr.update(choices=["(Semua)"] + vals, value="(Semua)")
|
|
|
|
| 973 |
|
|
|
|
|
|
|
|
|
|
| 974 |
|
| 975 |
with gr.Blocks() as demo:
|
| 976 |
+
gr.Markdown(f"""
|
| 977 |
+
# IPLM 2025 β Indeks FINAL (Penalti Coverage 68%) + Bell Curve + Analisis LLM (Word)
|
|
|
|
|
|
|
|
|
|
| 978 |
|
| 979 |
+
**Mode: NO UPLOAD (cache aktif).**
|
| 980 |
+
File dibaca dari server/repo:
|
| 981 |
+
- `DATA_FILE` = **{DATA_FILE}**
|
| 982 |
+
- `POP_KAB` = **{POP_KAB}**
|
| 983 |
+
- `POP_PROV` = **{POP_PROV}**
|
| 984 |
+
""")
|
| 985 |
+
|
| 986 |
+
state_df = gr.State(None)
|
| 987 |
+
state_pop_kab = gr.State(None)
|
| 988 |
+
state_pop_prov = gr.State(None)
|
| 989 |
+
state_meta = gr.State({})
|
| 990 |
+
|
| 991 |
+
with gr.Row():
|
| 992 |
+
btn_reload = gr.Button("Reload Data (paksa baca ulang file)")
|
| 993 |
+
info_box = gr.Markdown()
|
| 994 |
|
| 995 |
with gr.Row():
|
| 996 |
+
dd_prov = gr.Dropdown(label="Provinsi", choices=["(Semua)"], value="(Semua)")
|
| 997 |
+
dd_kab = gr.Dropdown(label="Kab/Kota", choices=["(Semua)"], value="(Semua)")
|
| 998 |
+
dd_kew = gr.Dropdown(label="Kewenangan", choices=["(Semua)"], value="(Semua)")
|
| 999 |
|
| 1000 |
+
dd_prov.change(fn=on_prov_change, inputs=[dd_prov], outputs=dd_kab)
|
| 1001 |
|
| 1002 |
run_btn = gr.Button("Jalankan Perhitungan")
|
| 1003 |
msg_out = gr.Markdown()
|
| 1004 |
|
| 1005 |
+
gr.Markdown("## Agregat (sub/dim + Indeks FINAL)")
|
| 1006 |
agg_out = gr.DataFrame(interactive=False)
|
| 1007 |
|
| 1008 |
+
gr.Markdown("## Detail (sub/dim + Indeks FINAL)")
|
| 1009 |
detail_out = gr.DataFrame(interactive=False)
|
| 1010 |
|
| 1011 |
+
gr.Markdown("## Verifikasi Coverage & GAP menuju 68% (kontrol mutu) β tanpa angka koma")
|
| 1012 |
verif_out = gr.DataFrame(interactive=False)
|
| 1013 |
|
| 1014 |
gr.Markdown("## Bell Curve Indeks FINAL β Semua Perpustakaan")
|
| 1015 |
bell_all = gr.Plot()
|
| 1016 |
|
| 1017 |
+
gr.Markdown("## Bell Curve Indeks FINAL β Sekolah")
|
| 1018 |
bell_sek = gr.Plot()
|
|
|
|
|
|
|
| 1019 |
|
| 1020 |
+
gr.Markdown("## Bell Curve Indeks FINAL β Umum")
|
| 1021 |
+
bell_um = gr.Plot()
|
| 1022 |
+
|
| 1023 |
+
gr.Markdown("## Bell Curve Indeks FINAL β Khusus")
|
| 1024 |
+
bell_kh = gr.Plot()
|
| 1025 |
+
|
| 1026 |
+
gr.Markdown("## Analisis Otomatis (LLM)")
|
| 1027 |
+
analysis_out = gr.Markdown()
|
| 1028 |
+
|
| 1029 |
+
# DOWNLOAD-ONLY (tanpa upload area)
|
| 1030 |
with gr.Row():
|
| 1031 |
+
agg_dl = gr.DownloadButton(label="Download Agregat (.xlsx)")
|
| 1032 |
+
det_dl = gr.DownloadButton(label="Download Detail (.xlsx)")
|
| 1033 |
+
ver_dl = gr.DownloadButton(label="Download Verifikasi Coverage (.xlsx)")
|
| 1034 |
+
word_dl = gr.DownloadButton(label="Download Analisis Word (.docx)")
|
| 1035 |
|
| 1036 |
run_btn.click(
|
| 1037 |
+
fn=run_calc,
|
| 1038 |
+
inputs=[dd_prov, dd_kab, dd_kew, state_df, state_pop_kab, state_pop_prov, state_meta],
|
| 1039 |
+
outputs=[
|
| 1040 |
+
agg_out, detail_out, verif_out,
|
| 1041 |
+
agg_dl, det_dl, ver_dl, word_dl,
|
| 1042 |
+
bell_all, bell_sek, bell_um, bell_kh,
|
| 1043 |
+
msg_out, analysis_out
|
| 1044 |
+
]
|
| 1045 |
+
)
|
| 1046 |
+
|
| 1047 |
+
demo.load(
|
| 1048 |
+
fn=lambda: ui_load(force=False),
|
| 1049 |
+
inputs=[],
|
| 1050 |
+
outputs=[state_df, state_pop_kab, state_pop_prov, state_meta, info_box, dd_prov, dd_kab, dd_kew]
|
| 1051 |
+
)
|
| 1052 |
+
|
| 1053 |
+
btn_reload.click(
|
| 1054 |
+
fn=lambda: ui_load(force=True),
|
| 1055 |
+
inputs=[],
|
| 1056 |
+
outputs=[state_df, state_pop_kab, state_pop_prov, state_meta, info_box, dd_prov, dd_kab, dd_kew]
|
| 1057 |
)
|
| 1058 |
|
| 1059 |
demo.launch()
|