Update app.py

app.py

@@ -1,5 +1,5 @@
-# app.py — versi super simpel ala ChatGPT
-import os, re, json, pickle, hashlib
+# app.py — RAG luwes untuk IPLM
+import os, re, json, hashlib
 from pathlib import Path
 import gradio as gr
 import numpy as np
@@ -7,87 +7,288 @@ from sklearn.neighbors import NearestNeighbors
 from sentence_transformers import SentenceTransformer
 
 # ========== Konfigurasi ==========
-DATA_PATH  = Path(os.getenv("DATA_PATH", "IPLM_QnA_Chatbot.jsonl"))
-EMB_MODEL  = os.getenv("EMB_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
-LOCAL_MODEL= os.getenv("LOCAL_MODEL", "google/gemma-2b-it")  # model lokal gratis & ringan
-TOP_K      = int(os.getenv("TOP_K", "4"))
-TEMPERATURE= float(os.getenv("TEMPERATURE", "0.2"))
-MAX_TOKENS = int(os.getenv("MAX_TOKENS", "256"))
-THRESHOLD  = float(os.getenv("THRESHOLD", "0.6"))
-
-SYSTEM_PROMPT = (
-    "You are an Indonesian librarian assistant. "
-    "Jawab singkat, akurat, dan sopan. "
-    "Jawab HANYA berdasarkan konteks yang diberikan. "
-    "Jika tidak ada jawabannya, balas persis: Data tidak tersedia."
-)
-
-# ========== Utils ==========
+DATA_PATH   = Path(os.getenv("DATA_PATH", "IPLM_QnA_Chatbot.jsonl"))
+EMB_MODEL   = os.getenv("EMB_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
+LOCAL_MODEL = os.getenv("LOCAL_MODEL", "google/gemma-2b-it")  # lokal & ringan
+TOP_K       = int(os.getenv("TOP_K", "5"))
+TEMPERATURE = float(os.getenv("TEMPERATURE", "0.4"))
+MAX_TOKENS  = int(os.getenv("MAX_TOKENS", "320"))
+THRESHOLD   = float(os.getenv("THRESHOLD", "0.62"))  # naikkan sedikit agar lebih tepercaya
+
+# ========== Prompt (lebih natural) ==========
+SYSTEM_PROMPT = """
+Kamu adalah asisten pustakawan Perpustakaan Nasional RI untuk topik IPLM (Indeks Pembangunan Literasi Masyarakat).
+Tugasmu:
+- Jawab hanya berdasarkan KONTEKS yang diberikan (jangan menambah fakta baru).
+- Tulis dengan bahasa Indonesia yang alami, ramah, dan mudah dipahami publik.
+- Jelaskan dengan contoh singkat bila membantu.
+- Jika konteks tidak cukup, katakan dengan jelas apa yang belum tersedia dan berikan langkah/arah yang bisa dilakukan.
+
+Format jawaban:
+1) Paragraf inti (1–3 kalimat) sesuai gaya diminta pengguna.
+2) Jika perlu, tambahkan poin-poin ringkas (maks 4 bullet) untuk memudahkan.
+3) Jika benar-benar tidak ada datanya di konteks, tulis: "Maaf, datanya belum tersedia di dasar informasi kami."
+"""
+
+# ========== Utilitas ==========
 def norm(s): return re.sub(r"\s+"," ",str(s or "").strip())
-def dataset_hash(rows):
-    m=hashlib.md5()
-    for r in rows: m.update((r["question"]+"|"+r["answer"]).encode())
-    return m.hexdigest()
-
-def load_jsonl(path:Path):
-    rows=[]
-    with path.open("r",encoding="utf-8") as f:
+
+def load_jsonl_with_variants(path: Path):
+    """
+    Mendukung skema:
+    - {"question": "...", "answer": "...", "q_variants": [...], "followups": [...], "source": "..."}
+    Kolom opsional: q_variants, followups, source
+    Jika q_variants tidak ada, pakai question saja.
+    """
+    items = []
+    with path.open("r", encoding="utf-8") as f:
         for line in f:
-            if not line.strip(): continue
-            obj=json.loads(line)
-            q=obj.get("question") or obj.get("q")
-            a=obj.get("answer")   or obj.get("a")
-            if q and a: rows.append({"question":norm(q),"answer":norm(a)})
-    return rows
-
-# ========== Retriever ==========
+            if not line.strip(): 
+                continue
+            obj = json.loads(line)
+            q = obj.get("question") or obj.get("q")
+            a = obj.get("answer")   or obj.get("a")
+            if not (q and a):
+                continue
+            qv = obj.get("q_variants") or []
+            if not isinstance(qv, list):
+                qv = [qv]
+            variants = [norm(q)] + [norm(x) for x in qv if x]
+            followups = obj.get("followups") or []
+            if not isinstance(followups, list):
+                followups = []
+            items.append({
+                "question": norm(q),
+                "answer": norm(a),
+                "q_variants": variants,
+                "followups": followups,
+                "source": norm(obj.get("source") or "")
+            })
+    return items
+
+# ========== Indexer/Retriever ==========
 class FAQIndex:
-    def __init__(self): self.rows=None; self.model=None; self.nn=None; self.emb=None
-    def build(self,rows):
-        self.rows=rows
-        self.model=SentenceTransformer(EMB_MODEL)
-        qs=[r["question"] for r in rows]
-        self.emb=self.model.encode(qs,normalize_embeddings=True,convert_to_numpy=True,show_progress_bar=False)
-        self.nn=NearestNeighbors(n_neighbors=min(10,len(qs)),metric="cosine").fit(self.emb)
-    def retrieve(self,query,top_k=TOP_K):
-        qv=self.model.encode([query],normalize_embeddings=True,convert_to_numpy=True,show_progress_bar=False)
-        d,i=self.nn.kneighbors(qv,n_neighbors=min(top_k,len(self.rows)))
-        sims=1.0-d[0]
-        return [{"question":self.rows[int(ix)]["question"],"answer":self.rows[int(ix)]["answer"],"score":float(s)} for ix,s in zip(i[0],sims)]
-
-faq=FAQIndex()
-faq.build(load_jsonl(DATA_PATH))
-
-# ========== Local LLM ==========
-_local_pipe=None
-def call_local(prompt):
+    def __init__(self, emb_model: str):
+        self.model_name = emb_model
+        self.model = None
+        self.rows = []            # setiap row = 1 QA
+        self.flat_q = []          # daftar semua query variants
+        self.parent = []          # mapping flat_q -> index row induk
+        self.nn = None
+        self.emb = None
+
+    def build(self, rows):
+        self.rows = rows
+        self.model = SentenceTransformer(self.model_name)
+        self.flat_q, self.parent = [], []
+        for i, r in enumerate(rows):
+            for qv in r["q_variants"]:
+                self.flat_q.append(qv)
+                self.parent.append(i)
+        self.emb = self.model.encode(
+            self.flat_q, 
+            normalize_embeddings=True, 
+            convert_to_numpy=True, 
+            show_progress_bar=False
+        )
+        self.nn = NearestNeighbors(
+            n_neighbors=min(15, len(self.flat_q)), metric="cosine"
+        ).fit(self.emb)
+
+    def retrieve(self, query: str, top_k=TOP_K):
+        if not self.flat_q:
+            return []
+        qv = self.model.encode(
+            [query], normalize_embeddings=True, convert_to_numpy=True, show_progress_bar=False
+        )
+        d, idx = self.nn.kneighbors(qv, n_neighbors=min(top_k, len(self.flat_q)))
+        sims = 1.0 - d[0]
+        hits = []
+        for ix, s in zip(idx[0], sims):
+            parent_i = self.parent[int(ix)]
+            base = self.rows[parent_i]
+            hits.append({
+                "match_q": self.flat_q[int(ix)],
+                "score": float(s),
+                "question": base["question"],
+                "answer": base["answer"],
+                "followups": base.get("followups") or [],
+                "source": base.get("source") or ""
+            })
+        # deduplicate by canonical question, keep best score
+        best = {}
+        for h in hits:
+            key = h["question"]
+            if key not in best or h["score"] > best[key]["score"]:
+                best[key] = h
+        hits_dedup = sorted(best.values(), key=lambda x: -x["score"])[:top_k]
+        return hits_dedup
+
+# ========== Local LLM (opsional rephrasing/merging) ==========
+_local_pipe = None
+def call_local_llm(prompt: str):
+    """
+    Jika lingkungan tidak punya model lokal, Anda bisa mematikan fungsi ini
+    dan langsung pakai template jawaban tanpa LLM (rule-based rephrase).
+    """
     global _local_pipe
-    from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-    import torch
-    if _local_pipe is None:
-        tok=AutoTokenizer.from_pretrained(LOCAL_MODEL)
-        mdl=AutoModelForCausalLM.from_pretrained(LOCAL_MODEL,torch_dtype=torch.float32)
-        _local_pipe=pipeline("text-generation",model=mdl,tokenizer=tok,device=-1)
-    out=_local_pipe(prompt,max_new_tokens=MAX_TOKENS,do_sample=True,temperature=TEMPERATURE)
-    return out[0]["generated_text"]
-
-# ========== Orchestrator ==========
-def answer_query(msg,history):
-    hits=faq.retrieve(msg,TOP_K)
-    if not hits: return "Data tidak tersedia."
-    if hits[0]["score"]>=THRESHOLD:
-        return hits[0]["answer"]
-    ctx="\n".join([f"- {h['answer']}" for h in hits])
-    prompt=f"{SYSTEM_PROMPT}\n\nKONTEKS:\n{ctx}\n\nPERTANYAAN: {msg}\n\nJAWAB:"
-    return call_local(prompt)
-
-# ========== UI Chat Only ==========
-demo=gr.ChatInterface(
-    fn=answer_query,
-    title="📚 IPLM Chatbot",
-    description="Tanya apa saja tentang IPLM. Jawaban hanya berdasarkan data JSONL.",
-    examples=["Apa itu IPLM?","Bagaimana menghitung IPLM?","Apa saja dimensi IPLM?"]
-)
-
-if __name__=="__main__":
+    try:
+        from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+        import torch
+        if _local_pipe is None:
+            tok = AutoTokenizer.from_pretrained(LOCAL_MODEL)
+            mdl = AutoModelForCausalLM.from_pretrained(LOCAL_MODEL, torch_dtype=torch.float32)
+            _local_pipe = pipeline("text-generation", model=mdl, tokenizer=tok, device=-1)
+        out = _local_pipe(
+            prompt, 
+            max_new_tokens=MAX_TOKENS, 
+            do_sample=True, 
+            temperature=TEMPERATURE,
+            pad_token_id=_local_pipe.tokenizer.eos_token_id
+        )
+        return out[0]["generated_text"]
+    except Exception as e:
+        # fallback: jika LLM gagal, kembalikan prompt terakhir (akan dipotong di caller)
+        return f"[LLM unavailable] {prompt}"
+
+# ========== Orchestration ==========
+STYLE_GUIDE = {
+    "Formal": "Nada formal, jelas, dan bernuansa kebijakan publik.",
+    "Santai": "Nada bersahabat dan ringan, hindari jargon teknis.",
+    "Ringkas": "Jawaban sangat singkat (1–2 kalimat) namun informatif.",
+    "Naratif": "Gaya bercerita singkat agar mudah dibayangkan."
+}
+
+def craft_prompt(context_bullets, question, style):
+    style_rule = STYLE_GUIDE.get(style, STYLE_GUIDE["Formal"])
+    ctx = "\n".join([f"- {c}" for c in context_bullets if c.strip()])
+    return f"""{SYSTEM_PROMPT}
+
+GAYA JAWABAN: {style_rule}
+
+KONTEKS:
+{ctx}
+
+PERTANYAAN PENGGUNA:
+{question}
+
+TULIS JAWABAN SEKARANG:
+"""
+
+def merge_context(hits):
+    # Ambil 3–5 jawaban teratas sebagai konteks bullet
+    bullets = []
+    for h in hits[:5]:
+        bullets.append(h["answer"])
+    return bullets
+
+def safe_cut(text, marker="TULIS JAWABAN SEKARANG:"):
+    # Jika pipeline mengembalikan prompt+jawaban, potong bagian setelah marker
+    if marker in text:
+        return text.split(marker, 1)[-1].strip()
+    return text.strip()
+
+def render_followups(hits, max_items=4):
+    # Kumpulkan followups dari hit terbaik
+    seen, out = set(), []
+    for h in hits:
+        for f in h.get("followups") or []:
+            f = norm(f)
+            if f and f not in seen:
+                out.append(f)
+                seen.add(f)
+            if len(out) >= max_items:
+                break
+        if len(out) >= max_items:
+            break
+    return out
+
+# ========== Build index ==========
+faq = FAQIndex(EMB_MODEL)
+faq.build(load_jsonl_with_variants(DATA_PATH))
+
+# ========== Gradio Callback ==========
+def answer_query(msg, chat_history, style, show_sources):
+    msg = norm(msg)
+    if not msg:
+        return "Silakan tulis pertanyaan tentang IPLM."
+
+    hits = faq.retrieve(msg, TOP_K)
+    if not hits:
+        return "Maaf, datanya belum tersedia di dasar informasi kami."
+
+    # Jika ada hit yang sangat kuat, pakai jawabannya langsung tapi tetap dipoles
+    top = hits[0]
+    if top["score"] >= THRESHOLD:
+        base = top["answer"]
+        # Poles ringan tanpa LLM
+        if style == "Ringkas":
+            final = base
+        elif style == "Santai":
+            final = f"Singkatnya, {base[0].lower()}{base[1:]}"
+        elif style == "Naratif":
+            final = f"Bayangkan kita menilai literasi di daerah. {base}"
+        else:
+            final = base
+
+        if show_sources:
+            meta = f"\n\n— Cocokkan dengan: “{top['question']}” • keyakinan ~{top['score']:.2f}"
+            if top.get("source"):
+                meta += f" • sumber: {top['source']}"
+            final += meta
+        # Tambah followups
+        fups = render_followups(hits)
+        if fups:
+            final += "\n\nCoba juga:\n" + "\n".join([f"- {x}" for x in fups])
+        return final
+
+    # Kalau skor belum mantap, gabungkan konteks lalu minta LLM memformulasikan jawaban luwes
+    ctx = merge_context(hits)
+    prompt = craft_prompt(ctx, msg, style)
+    raw = call_local_llm(prompt)
+    ans = safe_cut(raw)
+
+    # Proteksi: jika LLM malah halu/keluar jalur, fallback ke ringkasan rule-based
+    if not ans or "Maaf" in ans and "tidak" in ans and "tersedia" in ans:
+        # ringkasan sederhana dari konteks
+        ans = ctx[0] if ctx else "Maaf, datanya belum tersedia di dasar informasi kami."
+
+    if show_sources:
+        src_lines = []
+        for h in hits[:3]:
+            s = f'• “{h["question"]}” (keyakinan ~{h["score"]:.2f})'
+            if h.get("source"):
+                s += f' — sumber: {h["source"]}'
+            src_lines.append(s)
+        if src_lines:
+            ans += "\n\nRujukan terdekat:\n" + "\n".join(src_lines)
+
+    # Tambah saran follow-up
+    fups = render_followups(hits)
+    if fups:
+        ans += "\n\nCoba juga:\n" + "\n".join([f"- {x}" for x in fups])
+
+    return ans
+
+# ========== UI ==========
+with gr.Blocks(title="📚 IPLM Chatbot (luwes)") as demo:
+    gr.Markdown("## 📚 IPLM Chatbot\nTanya apa saja tentang IPLM. Jawaban berbasis data JSONL, disajikan dengan bahasa yang lebih luwes.")
+    with gr.Row():
+        style = gr.Radio(choices=list(STYLE_GUIDE.keys()), value="Formal", label="Gaya jawaban")
+        show_sources = gr.Checkbox(value=True, label="Tampilkan rujukan terdekat")
+    chat = gr.ChatInterface(
+        fn=lambda m,h: answer_query(m, h, style.value, show_sources.value),
+        title="IPLM Chatbot",
+        description="Jawaban hanya berdasarkan data JSONL, namun ditulis dengan gaya bahasa yang lebih natural.",
+        examples=[
+            "Sederhananya, apa itu IPLM?",
+            "Gimana cara hitung nilai IPLM biar jadi angka 0–100?",
+            "Bedanya dimensi kepatuhan sama kinerja apa ya?",
+            "Kalau anggaran BOS, yang dihitung bagian mana?",
+            "Siapa yang ngumpulin data di daerah dan gimana verifikasinya?"
+        ],
+        cache_examples=False
+    )
+
+if __name__ == "__main__":
     demo.launch()