Update app.py

app.py

@@ -1,4 +1,4 @@
-# app.py — IPLM Chatbot (UI sederhana ala GPT)
+# app.py — versi super simpel ala ChatGPT
 import os, re, json, pickle, hashlib
 from pathlib import Path
 import gradio as gr
@@ -6,160 +6,88 @@ import numpy as np
 from sklearn.neighbors import NearestNeighbors
 from sentence_transformers import SentenceTransformer
 
-# =================== Konfigurasi lewat ENV ===================
+# ========== Konfigurasi ==========
 DATA_PATH  = Path(os.getenv("DATA_PATH", "IPLM_QnA_Chatbot.jsonl"))
 EMB_MODEL  = os.getenv("EMB_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
-LOCAL_MODEL= os.getenv("LOCAL_MODEL", "microsoft/phi-2")  # model lokal (CPU)
+LOCAL_MODEL= os.getenv("LOCAL_MODEL", "google/gemma-2b-it")  # model lokal gratis & ringan
 TOP_K      = int(os.getenv("TOP_K", "4"))
 TEMPERATURE= float(os.getenv("TEMPERATURE", "0.2"))
 MAX_TOKENS = int(os.getenv("MAX_TOKENS", "256"))
-THRESHOLD  = float(os.getenv("THRESHOLD", "0.60"))  # ambil jawaban langsung jika skor >= threshold
-SHOW_SOURCES = os.getenv("SHOW_SOURCES", "false").lower() == "true"  # set true jika ingin tampilkan sumber terdekat
+THRESHOLD  = float(os.getenv("THRESHOLD", "0.6"))
 
 SYSTEM_PROMPT = (
-    "You are an Indonesian librarian assistant. Jawab singkat, akurat, dan sopan. "
-    "Jawab HANYA berdasarkan konteks yang diberikan. Jika tidak ada jawabannya, "
-    "balas persis: Data tidak tersedia."
+    "You are an Indonesian librarian assistant. "
+    "Jawab singkat, akurat, dan sopan. "
+    "Jawab HANYA berdasarkan konteks yang diberikan. "
+    "Jika tidak ada jawabannya, balas persis: Data tidak tersedia."
 )
 
-# =================== Utilitas ===================
-def norm(s: str) -> str:
-    if s is None: return ""
-    return re.sub(r"\s+", " ", str(s).strip())
-
-def dataset_hash(rows) -> str:
-    m = hashlib.md5()
-    for r in rows:
-        m.update((norm(r.get("question","")) + "|" + norm(r.get("answer",""))).encode("utf-8"))
+# ========== Utils ==========
+def norm(s): return re.sub(r"\s+"," ",str(s or "").strip())
+def dataset_hash(rows):
+    m=hashlib.md5()
+    for r in rows: m.update((r["question"]+"|"+r["answer"]).encode())
     return m.hexdigest()
 
-def load_jsonl(path: Path):
-    if not path.exists():
-        raise FileNotFoundError(f"JSONL tidak ditemukan: {path.resolve()}")
-    rows = []
-    with path.open("r", encoding="utf-8") as f:
+def load_jsonl(path:Path):
+    rows=[]
+    with path.open("r",encoding="utf-8") as f:
         for line in f:
             if not line.strip(): continue
-            obj = json.loads(line)
-            q = obj.get("question") or obj.get("pertanyaan") or obj.get("q")
-            a = obj.get("answer")   or obj.get("jawaban")    or obj.get("a")
-            if q and a: rows.append({"question": norm(q), "answer": norm(a)})
-    if not rows:
-        raise ValueError("JSONL kosong atau tidak ada pasangan 'question'/'answer'.")
-    # dedup by question
-    seen, uniq = set(), []
-    for r in rows:
-        if r["question"] in seen: continue
-        seen.add(r["question"]); uniq.append(r)
-    return uniq
+            obj=json.loads(line)
+            q=obj.get("question") or obj.get("q")
+            a=obj.get("answer")   or obj.get("a")
+            if q and a: rows.append({"question":norm(q),"answer":norm(a)})
+    return rows
 
-# =================== Retriever ===================
+# ========== Retriever ==========
 class FAQIndex:
-    def __init__(self):
-        self.rows=None; self.model=None; self.emb=None; self.nn=None
-
-    def build(self, rows, force=False):
-        cache_emb  = Path("embeddings.pkl")
-        cache_meta = Path("meta.json")
-        self.rows = rows
-        if not force and cache_emb.exists() and cache_meta.exists():
-            try:
-                meta = json.loads(cache_meta.read_text(encoding="utf-8"))
-                if meta.get("hash")==dataset_hash(rows) and meta.get("emb_model")==EMB_MODEL:
-                    cached = pickle.loads(cache_emb.read_bytes())
-                    self.emb, self.nn = cached["emb"], cached["nn"]
-                    if self.model is None: self.model = SentenceTransformer(EMB_MODEL)
-                    return
-            except Exception:
-                pass
-        self.model = SentenceTransformer(EMB_MODEL)
-        qs = [r["question"] for r in rows]
-        self.emb = self.model.encode(qs, normalize_embeddings=True, convert_to_numpy=True, show_progress_bar=False)
-        self.nn  = NearestNeighbors(n_neighbors=min(10, len(qs)), metric="cosine").fit(self.emb)
-        cache_emb.write_bytes(pickle.dumps({"emb": self.emb, "nn": self.nn}))
-        cache_meta.write_text(json.dumps({"hash": dataset_hash(rows), "emb_model": EMB_MODEL}, ensure_ascii=False))
-
-    def retrieve(self, query: str, top_k: int):
-        if not query.strip(): return []
-        qv = self.model.encode([query], normalize_embeddings=True, convert_to_numpy=True, show_progress_bar=False)
-        dists, idxs = self.nn.kneighbors(qv, n_neighbors=min(top_k, len(self.rows)))
-        sims = 1.0 - dists[0]
-        out = []
-        for i, sim in zip(idxs[0], sims):
-            r = self.rows[int(i)]
-            out.append({"question": r["question"], "answer": r["answer"], "score": float(sim)})
-        return out
-
-# =================== Local LLM (CPU) ===================
-_local_pipe = None
-def generate_with_local(prompt: str, temperature=TEMPERATURE, max_tokens=MAX_TOKENS):
+    def __init__(self): self.rows=None; self.model=None; self.nn=None; self.emb=None
+    def build(self,rows):
+        self.rows=rows
+        self.model=SentenceTransformer(EMB_MODEL)
+        qs=[r["question"] for r in rows]
+        self.emb=self.model.encode(qs,normalize_embeddings=True,convert_to_numpy=True,show_progress_bar=False)
+        self.nn=NearestNeighbors(n_neighbors=min(10,len(qs)),metric="cosine").fit(self.emb)
+    def retrieve(self,query,top_k=TOP_K):
+        qv=self.model.encode([query],normalize_embeddings=True,convert_to_numpy=True,show_progress_bar=False)
+        d,i=self.nn.kneighbors(qv,n_neighbors=min(top_k,len(self.rows)))
+        sims=1.0-d[0]
+        return [{"question":self.rows[int(ix)]["question"],"answer":self.rows[int(ix)]["answer"],"score":float(s)} for ix,s in zip(i[0],sims)]
+
+faq=FAQIndex()
+faq.build(load_jsonl(DATA_PATH))
+
+# ========== Local LLM ==========
+_local_pipe=None
+def call_local(prompt):
     global _local_pipe
-    try:
-        if _local_pipe is None:
-            import torch
-            from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-            tok   = AutoTokenizer.from_pretrained(LOCAL_MODEL)
-            model = AutoModelForCausalLM.from_pretrained(LOCAL_MODEL, torch_dtype=torch.float32)
-            _local_pipe = pipeline("text-generation", model=model, tokenizer=tok, device=-1)  # CPU
-        outs = _local_pipe(prompt, do_sample=True, temperature=float(temperature),
-                           max_new_tokens=int(max_tokens), return_full_text=False)
-        if isinstance(outs, list) and outs and "generated_text" in outs[0]:
-            return outs[0]["generated_text"]
-        return str(outs)
-    except Exception as e:
-        return f"❌ Gagal menjalankan model lokal: {e}"
-
-# =================== RAG (deterministic → generatif bila perlu) ===================
-def build_context(hits):
-    return "\n\n".join([f"[DOC {i}] {h['answer']}" for i, h in enumerate(hits, 1)])
-
-def answer_query(user_msg: str) -> str:
-    hits = faq.retrieve(user_msg, top_k=TOP_K)
-    if not hits:
-        return "Data tidak tersedia."
-
-    # Deterministic: kalau yakin → pakai jawaban sumber
-    if hits[0]["score"] >= THRESHOLD:
-        result = hits[0]['answer']
-        if SHOW_SOURCES:
-            bullets = "\n".join([f"- ({h['score']:.2f}) {h['question']}" for h in hits])
-            result += f"\n\n**Sumber terdekat:**\n{bullets}"
-        return result
-
-    # Jika kurang yakin → rangkum dengan LLM lokal
-    context = build_context(hits)
-    prompt = (
-        f"SISTEM: {SYSTEM_PROMPT}\n\n"
-        f"KONTEKS:\n{context}\n\n"
-        f"PERTANYAAN:\n{user_msg}\n\n"
-        "Instruksi: Jawab singkat dan HANYA berdasarkan KONTEKS di atas. "
-        "Jika tidak ada jawabannya, balas persis: Data tidak tersedia."
-    )
-    result = generate_with_local(prompt, temperature=TEMPERATURE, max_tokens=MAX_TOKENS)
-    if SHOW_SOURCES:
-        bullets = "\n".join([f"- ({h['score']:.2f}) {h['question']}" for h in hits])
-        result += f"\n\n**Sumber terdekat (lokal):**\n{bullets}"
-    return result
-
-# =================== Load data & index ===================
-faq = FAQIndex()
-_rows = load_jsonl(DATA_PATH)
-faq.build(_rows, force=False)
-
-# =================== UI minimal ===================
-def chat_fn(message, history):
-    return answer_query(message)
-
-with gr.Blocks(title="IPLM Chatbot") as demo:
-    gr.Markdown("### 📚 IPLM Chatbot\nTanya apa saja tentang **IPLM**. (UI sengaja disederhanakan)")
-    gr.ChatInterface(
-        fn=chat_fn,
-        title="",
-        description="",
-        examples=["Apa itu IPLM?", "Bagaimana menghitung IPLM?", "Apa saja dimensi IPLM?"],
-        cache_examples=False,
-        autofocus=True,
-    )
+    from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+    import torch
+    if _local_pipe is None:
+        tok=AutoTokenizer.from_pretrained(LOCAL_MODEL)
+        mdl=AutoModelForCausalLM.from_pretrained(LOCAL_MODEL,torch_dtype=torch.float32)
+        _local_pipe=pipeline("text-generation",model=mdl,tokenizer=tok,device=-1)
+    out=_local_pipe(prompt,max_new_tokens=MAX_TOKENS,do_sample=True,temperature=TEMPERATURE)
+    return out[0]["generated_text"]
+
+# ========== Orchestrator ==========
+def answer_query(msg,history):
+    hits=faq.retrieve(msg,TOP_K)
+    if not hits: return "Data tidak tersedia."
+    if hits[0]["score"]>=THRESHOLD:
+        return hits[0]["answer"]
+    ctx="\n".join([f"- {h['answer']}" for h in hits])
+    prompt=f"{SYSTEM_PROMPT}\n\nKONTEKS:\n{ctx}\n\nPERTANYAAN: {msg}\n\nJAWAB:"
+    return call_local(prompt)
+
+# ========== UI Chat Only ==========
+demo=gr.ChatInterface(
+    fn=answer_query,
+    title="📚 IPLM Chatbot",
+    description="Tanya apa saja tentang IPLM. Jawaban hanya berdasarkan data JSONL.",
+    examples=["Apa itu IPLM?","Bagaimana menghitung IPLM?","Apa saja dimensi IPLM?"]
+)
 
-if __name__ == "__main__":
+if __name__=="__main__":
     demo.launch()