Update app.py

app.py

@@ -1,4 +1,21 @@
-import os, re, math, io
+# ============================================================
+# IPLM Audit — HF Spaces (Gradio) — + LLM Analysis (Optional)
+# - Scorecard + Benford + Similarity + Scatter
+# - Tambahan: LLM narasi untuk Scorecard (teknokratis)
+# ============================================================
+
+import os
+
+# ---- CRASH FIX (HF Spaces Exit 139 / SIGSEGV) ----
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["OPENBLAS_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
+os.environ["NUMEXPR_NUM_THREADS"] = "1"
+os.environ["MPLBACKEND"] = "Agg"
+os.environ["PYTHONUNBUFFERED"] = "1"
+
+import re, math, io, json, textwrap
 import numpy as np
 import pandas as pd
 import gradio as gr
@@ -7,13 +24,25 @@ from PIL import Image
 from scipy.stats import chisquare
 from sklearn.preprocessing import StandardScaler
 from sklearn.metrics.pairwise import cosine_similarity
+
+import matplotlib
+matplotlib.use("Agg")
 import matplotlib.pyplot as plt
 
 
 # ============================================================
 # CONFIG
 # ============================================================
-DATA_PATH = os.getenv("IPLM_DATA_PATH", "IPLM_clean_manual_131225.xlsx")
+DATA_PATH = os.getenv("IPLM_DATA_PATH", "data/IPLM_clean_manual_131225.xlsx")
+
+# LLM provider optional:
+# - OpenAI: set OPENAI_API_KEY (+ optional OPENAI_MODEL)
+# - Gemini: set GEMINI_API_KEY (+ optional GEMINI_MODEL)
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "").strip()
+OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")  # aman & murah (ubah bebas)
+
+GEMINI_API_KEY = os.getenv("HF_TOKEN_DQ", "").strip()
+GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-1.5-flash")
 
 EXCLUDE_COLS_EXACT = {"kontak_wa", "npp", "tanggal_kirim", "updated_at", "created_at"}
 
@@ -40,6 +69,14 @@ BENFORD_EXCLUDE_PATTERNS = [
 def canon(s: str) -> str:
     return re.sub(r"[^a-z0-9]+", "", str(s).lower())
 
+def clean_text_col(s: pd.Series) -> pd.Series:
+    s = s.astype(str)
+    s = s.str.replace("\u00a0", " ", regex=False)
+    s = s.str.replace(r"\s+", " ", regex=True)
+    s = s.str.strip()
+    s = s.mask(s.str.lower().isin(["nan", "none", "null", ""]), np.nan)
+    return s
+
 def pick_col(df, candidates):
     cols = list(df.columns)
     cc = {canon(c): c for c in cols}
@@ -56,21 +93,19 @@ def pick_col(df, candidates):
 
 def detect_geo_cols(df):
     prov = pick_col(df, ["provinsi", "propinsi", "province"])
-    kab  = pick_col(df, ["kab_kota", "kabkota", "kabupatenkota", "kabupaten/kota", "kabupaten", "kota", "regency", "city"])
+    kab  = pick_col(df, ["kab_kota", "kabkota", "kabupatenkota", "kabupaten/kota",
+                        "kabupaten", "kota", "regency", "city"])
     return prov, kab
 
 def detect_kewenangan_col(df):
-    return pick_col(df, ["kewenangan", "pu_level", "level_kewenangan", "kewenangan_pengelola", "kewenangan_perpustakaan", "level"])
+    return pick_col(df, ["kewenangan", "pu_level", "level_kewenangan",
+                        "kewenangan_pengelola", "kewenangan_perpustakaan", "level"])
 
 def load_excel(path):
     df = pd.read_excel(path, engine="openpyxl")
     for c in df.columns:
         if df[c].dtype == object:
-            df[c] = (df[c].astype(str)
-                     .str.replace("\u00a0", " ", regex=False)
-                     .str.replace(r"\s+", " ", regex=True)
-                     .str.strip())
-            df.loc[df[c].str.lower().isin(["nan", "none", "null", ""]), c] = np.nan
+            df[c] = clean_text_col(df[c])
     return df
 
 def clean_str_list(values):
@@ -91,7 +126,6 @@ def clean_str_list(values):
     return uniq
 
 def safe_numeric_cols(df, exclude=set(), min_non_na=0.25):
-    """Numeric cols used for completeness/zero/similarity. Hard-exclude columns by exact name."""
     hard = {canon(x) for x in EXCLUDE_COLS_EXACT}
     cols = []
     for c in df.columns:
@@ -174,10 +208,14 @@ def scatter_plot(peer_agg, x_col, y_col):
 
 
 # ============================================================
-# LOAD ONCE (GLOBAL)
+# LOAD DATA (GLOBAL)
 # ============================================================
 if not os.path.exists(DATA_PATH):
-    raise FileNotFoundError(f"Data file not found: {DATA_PATH}. Taruh excel di repo: data/..., atau set env IPLM_DATA_PATH.")
+    raise FileNotFoundError(
+        f"Data file not found: {DATA_PATH}\n"
+        "Taruh file excel di repo: data/IPLM_clean_manual_131225.xlsx\n"
+        "atau set env variable IPLM_DATA_PATH."
+    )
 
 df_raw = load_excel(DATA_PATH)
 prov_col, kab_col = detect_geo_cols(df_raw)
@@ -187,11 +225,9 @@ if prov_col is None or kab_col is None:
     raise ValueError("Kolom provinsi/kab_kota tidak terdeteksi. Pastikan ada kolom provinsi dan kab_kota.")
 
 df = df_raw.copy()
-df["_prov_str"] = df[prov_col].astype(str).str.strip()
-df["_kab_str"] = df[kab_col].astype(str).str.strip()
-df.loc[df["_prov_str"].str.lower().isin(["nan", "none", "null", ""]), "_prov_str"] = np.nan
-df.loc[df["_kab_str"].str.lower().isin(["nan", "none", "null", ""]), "_kab_str"] = np.nan
-df = df[df["_prov_str"].notna() & df["_kab_str"].notna()].copy()  # penting supaya tidak "campur"
+df["_prov_str"] = clean_text_col(df[prov_col])
+df["_kab_str"]  = clean_text_col(df[kab_col])
+df = df[df["_prov_str"].notna() & df["_kab_str"].notna()].copy()  # cegah mixing
 
 exclude_base = {prov_col, kab_col, "_prov_str", "_kab_str"}
 hard_exclude_cols_in_file = {c for c in df.columns if canon(c) in {canon(x) for x in EXCLUDE_COLS_EXACT}}
@@ -201,58 +237,69 @@ num_cols_all = safe_numeric_cols(df, exclude=exclude_base)
 benford_cols = [c for c in num_cols_all if is_benford_applicable(c)]
 
 PROVS = clean_str_list(df["_prov_str"].unique().tolist())
+if not PROVS:
+    raise ValueError("Tidak ada nilai provinsi yang valid setelah cleaning.")
 
-prov_cache_peer = {}  # cache per prov for similarity
+prov_cache_peer = {}  # cache peer per prov
 
 
 def kabs_for_prov(pv):
+    if pv is None:
+        return []
     return clean_str_list(df.loc[df["_prov_str"] == pv, "_kab_str"].unique().tolist())
 
 def kew_for(pv, kv):
     if not kew_col or kew_col not in df.columns:
         return ["(kewenangan tidak tersedia)"]
-    vals = clean_str_list(df.loc[(df["_prov_str"] == pv) & (df["_kab_str"] == kv), kew_col].dropna().unique().tolist())
+    if pv is None or kv is None:
+        return ["(pilih provinsi & kab/kota)"]
+    vals = clean_str_list(
+        df.loc[(df["_prov_str"] == pv) & (df["_kab_str"] == kv), kew_col]
+        .dropna().unique().tolist()
+    )
     return vals if vals else ["(kewenangan kosong)"]
 
 def get_peer_agg_for_prov(pv):
     if pv in prov_cache_peer:
         return prov_cache_peer[pv]
     peer = df[df["_prov_str"] == pv]
-    peer_agg = peer.groupby("_kab_str")[num_cols_all].apply(
-        lambda g: g.apply(pd.to_numeric, errors="coerce").mean()
-    ).reset_index().rename(columns={"_kab_str": "kab_kota"})
+    if peer.empty:
+        peer_agg = pd.DataFrame({"kab_kota": []})
+        prov_cache_peer[pv] = peer_agg
+        return peer_agg
+
+    peer_agg = (
+        peer.groupby("_kab_str")[num_cols_all]
+        .apply(lambda g: g.apply(pd.to_numeric, errors="coerce").mean())
+        .reset_index()
+        .rename(columns={"_kab_str": "kab_kota"})
+    )
     prov_cache_peer[pv] = peer_agg
     return peer_agg
 
 
 # ============================================================
-# CORE AUDIT FUNCTION (STRICT FILTER)
+# CORE AUDIT
 # ============================================================
 def audit(pv, kv, kw):
-    # strict filter: prov + kab (+ kewenangan if available & chosen)
-    dfx = df[(df["_prov_str"] == pv) & (df["_kab_str"] == kv)].copy()
+    if pv is None or kv is None:
+        return "❌ Pilih provinsi dan kab/kota.", pd.DataFrame(), pd.DataFrame(), None, None, pd.DataFrame()
 
-    if kew_col and kew_col in dfx.columns and kw and not kw.startswith("("):
-        dfx = dfx[dfx[kew_col].astype(str).str.strip() == kw].copy()
+    dfx = df[(df["_prov_str"] == pv) & (df["_kab_str"] == kv)].copy()
+    if kew_col and kew_col in dfx.columns and kw and not str(kw).startswith("("):
+        dfx = dfx[dfx[kew_col].astype(str).str.strip() == str(kw).strip()].copy()
 
     if dfx.empty:
-        return (
-            "❌ Data kosong setelah filter (cek kewenangan / validitas label).",
-            pd.DataFrame(),
-            pd.DataFrame(),
-            None,
-            None
-        )
+        return "❌ Data kosong setelah filter (cek kewenangan/label).", pd.DataFrame(), pd.DataFrame(), None, None, pd.DataFrame()
 
     if not num_cols_all:
-        return ("❌ Tidak ada kolom numerik yang cukup.", pd.DataFrame(), pd.DataFrame(), None, None)
+        return "❌ Tidak ada kolom numerik yang cukup.", pd.DataFrame(), pd.DataFrame(), None, None, pd.DataFrame()
 
     num_all = dfx[num_cols_all].apply(pd.to_numeric, errors="coerce")
-
     completeness = float(num_all.notna().mean().mean())
     zero_rate = float((num_all.fillna(0) == 0).mean().mean())
 
-    # Benford (applicable only, already excluded hard cols)
+    # Benford
     best = None
     rows = []
     for c in benford_cols:
@@ -270,12 +317,12 @@ def audit(pv, kv, kw):
         ben_note = f"Benford strongest: {best['kolom']} | n={best['n']} | MAD={best['mad']:.4f} ({benford_flag(best['mad'])}) | p={best['p_value']:.3g}"
         ben_img = benford_plot(best["obs"])
 
-    # Similarity (peer se-provinsi) => strict prov only (no mixing)
+    # Similarity within prov
     peer_agg = get_peer_agg_for_prov(pv)
     sim_tbl = pd.DataFrame()
     top_sim = None
 
-    if peer_agg.shape[0] >= 3:
+    if not peer_agg.empty and peer_agg.shape[0] >= 3:
         X = peer_agg[num_cols_all].replace([np.inf, -np.inf], np.nan).fillna(0.0).to_numpy(float)
         Xs = StandardScaler().fit_transform(X)
         sim = cosine_similarity(Xs)
@@ -295,20 +342,19 @@ def audit(pv, kv, kw):
             if not sim_tbl.empty:
                 top_sim = float(sim_tbl["cosine_similarity"].max())
 
-    # scatter
     scat_img = None
-    if peer_agg.shape[0] >= 3:
+    if not peer_agg.empty and peer_agg.shape[0] >= 3:
         vars_ = peer_agg[num_cols_all].replace([np.inf, -np.inf], np.nan).fillna(0.0).var(axis=0).sort_values(ascending=False)
         if len(vars_) >= 2 and vars_.iloc[0] > 0 and vars_.iloc[1] > 0:
-            x_col, y_col = vars_.index[0], vars_.index[1]
-            scat_img = scatter_plot(peer_agg, x_col, y_col)
+            scat_img = scatter_plot(peer_agg, vars_.index[0], vars_.index[1])
 
     too_perfect = (completeness > 0.98) and (zero_rate < 0.02)
 
     scorecard = pd.DataFrame([
         ["Provinsi", pv, ""],
         ["Kab/Kota", kv, ""],
-        ["Kewenangan", kw if kw else "NA", f"Sumber: {kew_col}" if (kew_col and not str(kw).startswith("(")) else "Kewenangan tidak tersedia/kosong."],
+        ["Kewenangan", kw if kw else "NA",
+         f"Sumber: {kew_col}" if (kew_col and not str(kw).startswith("(")) else "Kewenangan tidak tersedia/kosong."],
         ["Completeness (numeric)", f"{completeness:.2%}",
          "Kelengkapan tinggi; pastikan berasal dari validasi input (wajib isi) atau data administratif lengkap. Jika ada imputasi, dokumentasikan prosedurnya."],
         ["Zero-rate (numeric)", f"{zero_rate:.2%}",
@@ -330,7 +376,92 @@ def audit(pv, kv, kw):
 
 
 # ============================================================
-# GRADIO UI (DEPLOY READY)
+# LLM (OPTIONAL)
+# ============================================================
+def _llm_available():
+    return bool(OPENAI_API_KEY) or bool(GEMINI_API_KEY)
+
+def llm_analyze_scorecard(pv, kv, kw, scorecard_df, ben_tbl, sim_tbl):
+    """
+    Return a policy-style narrative in Indonesian.
+    Works if OPENAI_API_KEY or GEMINI_API_KEY is set.
+    """
+    if scorecard_df is None or len(scorecard_df) == 0:
+        return "❌ Scorecard kosong. Jalankan audit dulu."
+
+    if not _llm_available():
+        return (
+            "⚠️ **LLM belum aktif** karena API key belum diset.\n\n"
+            "Set salah satu environment variable di HuggingFace Space:\n"
+            "- `OPENAI_API_KEY` (opsional `OPENAI_MODEL`)\n"
+            "- atau `GEMINI_API_KEY` (opsional `GEMINI_MODEL`)\n"
+        )
+
+    payload = {
+        "provinsi": pv,
+        "kab_kota": kv,
+        "kewenangan": kw,
+        "scorecard": scorecard_df.to_dict(orient="records"),
+        "top_benford_signals": ben_tbl.head(10).to_dict(orient="records") if isinstance(ben_tbl, pd.DataFrame) else [],
+        "top_similarity": sim_tbl.head(10).to_dict(orient="records") if isinstance(sim_tbl, pd.DataFrame) else []
+    }
+
+    system = (
+        "Anda adalah analis kebijakan publik dan auditor kualitas data untuk indikator pembangunan literasi/perpustakaan. "
+        "Tulis analisis ringkas namun tajam, berbahasa Indonesia teknokratis (gaya Perpusnas/pemerintah). "
+        "Fokus pada interpretasi scorecard: kelengkapan, zero-rate, Benford, similarity, implikasi risiko, dan rekomendasi tindak lanjut "
+        "(cek bukti dukung, metadata, log input, sampling/validasi). Hindari asumsi yang tidak ada di data."
+    )
+
+    user = (
+        "Buatkan ANALISIS NARATIF untuk hasil audit berikut. Struktur wajib:\n"
+        "1) Ringkasan status (1 paragraf)\n"
+        "2) Interpretasi tiap komponen (bullet)\n"
+        "3) Rekomendasi tindak lanjut prioritas (maks 6 bullet)\n\n"
+        f"DATA (JSON):\n{json.dumps(payload, ensure_ascii=False)}"
+    )
+
+    # ---- Try OpenAI first if key exists ----
+    if OPENAI_API_KEY:
+        try:
+            from openai import OpenAI
+            client = OpenAI(api_key=OPENAI_API_KEY)
+            resp = client.chat.completions.create(
+                model=OPENAI_MODEL,
+                messages=[
+                    {"role": "system", "content": system},
+                    {"role": "user", "content": user},
+                ],
+                temperature=0.2,
+                max_tokens=700,
+            )
+            return resp.choices[0].message.content.strip()
+        except Exception as e:
+            err = str(e)
+            # fallback to gemini if available
+            if not GEMINI_API_KEY:
+                return f"❌ LLM error (OpenAI): {err}"
+            # else continue to Gemini
+
+    # ---- Gemini fallback ----
+    if GEMINI_API_KEY:
+        try:
+            from google import genai
+            client = genai.Client(api_key=GEMINI_API_KEY)
+            resp = client.models.generate_content(
+                model=GEMINI_MODEL,
+                contents=f"{system}\n\n{user}",
+            )
+            text = getattr(resp, "text", None)
+            return (text or "").strip() or "❌ Gemini tidak mengembalikan teks."
+        except Exception as e:
+            return f"❌ LLM error (Gemini): {e}"
+
+    return "❌ LLM tidak tersedia."
+
+
+# ============================================================
+# GRADIO UI
 # ============================================================
 def ui_init():
     pv = PROVS[0] if PROVS else None
@@ -352,10 +483,13 @@ def on_kab_change(pv, kv):
     kw = kews[0] if kews else None
     return gr.update(choices=kews, value=kw)
 
+# state holder for last outputs (to feed LLM button without recompute)
 def run_audit(pv, kv, kw):
     narasi, scorecard, ben_tbl, ben_img, scat_img, sim_tbl = audit(pv, kv, kw)
-    # Return order: markdown, scorecard df, benford df, benford img, scatter img, sim df
-    return narasi, scorecard, ben_tbl, ben_img, scat_img, sim_tbl
+    return narasi, scorecard, ben_tbl, ben_img, scat_img, sim_tbl, scorecard, ben_tbl, sim_tbl
+
+def run_llm(pv, kv, kw, scorecard_df, ben_tbl, sim_tbl):
+    return llm_analyze_scorecard(pv, kv, kw, scorecard_df, ben_tbl, sim_tbl)
 
 
 pv0, kv0, kw0, kabs0, kews0 = ui_init()
@@ -366,7 +500,8 @@ with gr.Blocks(title="IPLM Audit — Kualitas Data & Indikasi Tidak Wajar", them
         f"- Sumber data: `{DATA_PATH}`\n"
         f"- EXCLUDE (no analysis): `{', '.join(sorted(EXCLUDE_COLS_EXACT))}`\n"
         f"- prov_col = `{prov_col}` · kab_col = `{kab_col}` · kewenangan_col = `{kew_col if kew_col else 'TIDAK ADA'}`\n"
-        "---"
+        "---\n"
+        "**LLM Analysis (opsional):** set `OPENAI_API_KEY` atau `GEMINI_API_KEY` di Space Variables."
     )
 
     with gr.Row():
@@ -377,7 +512,9 @@ with gr.Blocks(title="IPLM Audit — Kualitas Data & Indikasi Tidak Wajar", them
     prov.change(on_prov_change, inputs=prov, outputs=[kab, kew], show_progress=False)
     kab.change(on_kab_change, inputs=[prov, kab], outputs=kew, show_progress=False)
 
-    btn = gr.Button("Run Audit", variant="primary")
+    with gr.Row():
+        btn = gr.Button("Run Audit", variant="primary")
+        btn_llm = gr.Button("Generate LLM Analysis", variant="secondary")
 
     out_md = gr.Markdown()
     out_score = gr.Dataframe(label="Scorecard", interactive=False, wrap=True)
@@ -389,6 +526,26 @@ with gr.Blocks(title="IPLM Audit — Kualitas Data & Indikasi Tidak Wajar", them
 
     out_sim = gr.Dataframe(label="Top Similarity (se-Provinsi)", interactive=False, wrap=True)
 
-    btn.click(run_audit, inputs=[prov, kab, kew], outputs=[out_md, out_score, out_ben_tbl, out_ben_img, out_scat_img, out_sim])
+    gr.Markdown("## Analisis Naratif (LLM)")
+    out_llm = gr.Markdown()
+
+    # hidden states: store last tables for llm button
+    st_score = gr.State(pd.DataFrame())
+    st_ben = gr.State(pd.DataFrame())
+    st_sim = gr.State(pd.DataFrame())
+
+    btn.click(
+        run_audit,
+        inputs=[prov, kab, kew],
+        outputs=[out_md, out_score, out_ben_tbl, out_ben_img, out_scat_img, out_sim, st_score, st_ben, st_sim],
+        show_progress=False
+    )
+
+    btn_llm.click(
+        run_llm,
+        inputs=[prov, kab, kew, st_score, st_ben, st_sim],
+        outputs=[out_llm],
+        show_progress=True
+    )
 
-demo.queue().launch()
+demo.launch()