commit

app.py

@@ -1,5 +1,6 @@
 # app.py
 import os
+import re
 import base64
 import gradio as gr
 from openai import OpenAI
@@ -8,30 +9,40 @@ from supabase_client import load_file_bytes
 from rag_pipeline import rag_answer
 
 client = OpenAI()
+
 BUCKET = os.environ["SUPABASE_BUCKET"]
 SUPABASE_URL = os.environ["SUPABASE_URL"]
 
-# ------------------------------------------
-# URLs cho Prüfungsordnung (PDF) + HG NRW
-# ------------------------------------------
-
-# PDF nằm trong Supabase (như trước)
 PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
+HG_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
 
-# ⚠️ Đây là link chính thức của Hochschulgesetz NRW trên recht.nrw.de
-HG_URL  = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
 
-# ------------------------------------------
-# Viewer PDF base64
-# ------------------------------------------
+# -------------------------------------------------------------------
+# PDF BASE64 để nhúng iframe
+# -------------------------------------------------------------------
 def encode_pdf_src():
     pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
     b64 = base64.b64encode(pdf_bytes).decode("utf-8")
     return f"data:application/pdf;base64,{b64}"
 
-# ------------------------------------------
-# Speech-to-text (Whisper)
-# ------------------------------------------
+
+# -------------------------------------------------------------------
+# CLEAN STT
+# -------------------------------------------------------------------
+FILLER = ["äh", "ähm", "uh", "hmm", "mmh", "ah", "oh", "also", "sozusagen", "halt"]
+
+
+def clean_transcript(t):
+    if not t:
+        return ""
+    t = t.lower().strip()
+    for f in FILLER:
+        t = re.sub(rf"\b{f}\b", "", t)
+    t = re.sub(r"[^a-zA-ZäöüÄÖÜß0-9,.? ]+", " ", t)
+    t = re.sub(r"\s+", " ", t).strip()
+    return t.capitalize()
+
+
 def transcribe(audio_path):
     if audio_path is None:
         return ""
@@ -39,95 +50,102 @@ def transcribe(audio_path):
         result = client.audio.transcriptions.create(
             model="whisper-1",
             file=f,
-            language="de",      # tiếng Đức
-            temperature=0.0
+            language="de",
+            temperature=0.0,
         )
-    return (result.text or "").strip()
-
-# ------------------------------------------
-# HÀM CHAT CHÍNH
-# ------------------------------------------
-def chat_fn(text, audio, history):
-    text = (text or "").strip()
-
-    # 1) Ưu tiên TEXT; chỉ dùng audio nếu không có text
-    if text:
-        question = text
-    elif audio is not None:
-        question = transcribe(audio)
-    else:
-        return history, "", None  # không có input
+    return clean_transcript(result.text or "")
+
+
+# -------------------------------------------------------------------
+# CHAT FUNCTION — KHÔNG ƯU TIÊN TEXT/AUDIO
+# User CHỌN mode: "text" hoặc "audio"
+# -------------------------------------------------------------------
+def chat_fn(mode, text, audio, history):
+    history = history or []
+
+    # --- MODE: TEXT ---
+    if mode == "text":
+        if not (text or "").strip():
+            return history, "Bitte Text eingeben.", None
+        question = text.strip()
+
+    # --- MODE: SPRACHE ---
+    if mode == "audio":
+        if audio is None:
+            return history, "Bitte ins Mikrofon sprechen.", None
 
-    if not question:
-        return history, "Spracherkennung fehlgeschlagen.", None
+        question = transcribe(audio)
+        if not question:
+            return history, "Spracherkennung fehlgeschlagen. Bitte erneut versuchen.", None
 
-    # 2) Gọi RAG
-    answer, docs = rag_answer(question, history or [])
+    # --- RAG ---
+    answer, docs = rag_answer(question, history)
 
-    # 3) Xây block Quellen ở dạng Markdown, DÙNG META từ docs
-    quellen_md_lines = ["", "### 📚 Quellen (verwendete Dokumentstellen):"]
+    # --- Quellen ---
+    quellen = ["", "### 📚 Quellen:"]
     for i, d in enumerate(docs):
-        meta = d.get("metadata", {}) or {}
-        src = meta.get("source", "?")
-        page = meta.get("page", None)
-        anchor_id = meta.get("anchor_id")
+        meta = d["metadata"]
+        src = meta.get("source")
 
-        # Prüfungsordnung (PDF) – nhảy đúng Seite
         if src.startswith("Prüfungsordnung"):
-            # trong ingest page lưu 1-based; nếu bạn dùng 0-based thì +1 ở đây
-            page_num = page if isinstance(page, int) else None
-            if page_num:
-                url = f"{PDF_URL}#page={page_num}"
-                title = f"Quelle {i+1} – {src}, Seite {page_num}"
-            else:
-                url = PDF_URL
-                title = f"Quelle {i+1} – {src}"
-        # Hochschulgesetz NRW – dùng URL chính thức + anchor_id (para)
+            page = meta.get("page")
+            url = f"{PDF_URL}#page={page}"
+            title = f"Quelle {i+1} – Prüfungsordnung, Seite {page}"
         else:
-            if anchor_id:
-                url = f"{HG_URL}#{anchor_id}"
-            else:
-                url = HG_URL
+            url = HG_URL
             title = f"Quelle {i+1} – Hochschulgesetz NRW"
 
-        snippet = (d.get("content") or "")[:200].replace("\n", " ")
-
-        quellen_md_lines.append(f"- [{title}]({url})")
-        quellen_md_lines.append(f"  > {snippet}")
+        snip = d["content"][:160].replace("\n", " ")
+        quellen.append(f"- [{title}]({url})")
+        quellen.append(f"  > {snip}")
 
-    quellen_md = "\n".join(quellen_md_lines)
+    bot_msg = answer + "\n\n" + "\n".join(quellen)
 
-    # 4) GỘP câu trả lời + Quellen vào NỘI DUNG CHATBOT
-    bot_msg = answer + "\n\n" + quellen_md
-
-    new_history = (history or []) + [
+    new_history = history + [
         {"role": "user", "content": question},
         {"role": "assistant", "content": bot_msg},
     ]
 
-    # Trả về history (hiển thị trong Chatbot) + block Markdown (nếu muốn xem riêng) + reset audio
     return new_history, bot_msg, gr.update(value=None)
 
-# ------------------------------------------
-# GIAO DIỆN
-# ------------------------------------------
+
+# -------------------------------------------------------------------
+# UI — GIỐNG HÌNH ĐÍNH KÈM
+# -------------------------------------------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("# ⚖️ Sprachbasierter Chatbot für Prüfungsrecht")
+
+    gr.Markdown("""
+    # ⚖️ Sprachbasierter Chatbot für Prüfungsrecht  
+    Wähle eine Eingabemethode: Text oder Sprache.
+    """)
 
     with gr.Row():
-        # Bên trái: Chat
+
+        # ======================
+        # LEFT SIDE: CHAT UI
+        # ======================
         with gr.Column(scale=3):
-            # Chatbot RENDER Markdown (type="messages")
-            chatbot = gr.Chatbot()
-            text_input = gr.Textbox(label="Frage eingeben")
+
+            chatbot = gr.Chatbot(label="Chatverlauf")
+
+            mode_select = gr.Radio(
+                ["text", "audio"],
+                value="text",
+                label="Eingabemodus",
+                info="Wähle zwischen Text oder Sprache",
+            )
+
+            text_input = gr.Textbox(label="Text eingeben")
             audio_input = gr.Audio(type="filepath", label="Spracheingabe (Mikrofon)")
-            send_btn = gr.Button("Senden")
 
-            # Preview Markdown của câu trả lời cuối (tuỳ chọn)
+            send_btn = gr.Button("Senden")
             answer_preview = gr.Markdown("")
 
-        # Bên phải: Viewer
+        # ======================
+        # RIGHT SIDE: VIEWER
+        # ======================
         with gr.Column(scale=2):
+
             gr.Markdown("### 📄 Prüfungsordnung (PDF)")
             gr.HTML(
                 f"<iframe src='{encode_pdf_src()}' width='100%' height='250' style='border:none;'></iframe>"
@@ -138,10 +156,9 @@ with gr.Blocks() as demo:
                 f"<iframe src='{HG_URL}' width='100%' height='250' style='border:none;'></iframe>"
             )
 
-    # Nút gửi
     send_btn.click(
         chat_fn,
-        inputs=[text_input, audio_input, chatbot],
+        inputs=[mode_select, text_input, audio_input, chatbot],
         outputs=[chatbot, answer_preview, audio_input],
     )
 

ingest.py

@@ -1,6 +1,7 @@
 # ingest.py
 import os
 from io import BytesIO
+
 from bs4 import BeautifulSoup
 from pypdf import PdfReader
 
@@ -9,14 +10,33 @@ from langchain_openai import OpenAIEmbeddings
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 
+# -------------------------------------------------------------------
+# ENV + URLs
+# -------------------------------------------------------------------
 BUCKET = os.environ["SUPABASE_BUCKET"]
 SUPABASE_URL = os.environ["SUPABASE_URL"]
 
+# Public URLs trong Supabase Storage (chỉ dùng để tham chiếu / Quelle)
 PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
-HG_URL  = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/hochschulgesetz.html"
+HG_STORAGE_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/hochschulgesetz.html"
+
+# (In App dùng link chính thức của HG NRW, còn đây chỉ để meta nếu cần)
+OFFICIAL_HG_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
 
 
+# -------------------------------------------------------------------
+# Loader PDF Prüfungsordnung
+# -------------------------------------------------------------------
 def load_pdf_docs():
+    """
+    PDF Prüfungsordnung:
+    - Đọc từ Supabase Storage
+    - Trích text từng trang
+    - Mỗi trang là 1 Document với metadata:
+        - source: "Prüfungsordnung (PDF)"
+        - page: SỐ TRANG 1-based (Seite 1, 2, 3, ...)
+        - pdf_url: URL public của PDF trong Supabase (không #page)
+    """
     pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
     reader = PdfReader(BytesIO(pdf_bytes))
 
@@ -24,39 +44,60 @@ def load_pdf_docs():
     for i, page in enumerate(reader.pages):
         text = page.extract_text() or ""
 
+        # Lưu page 1-based để sau dùng trực tiếp trong UI
+        page_num = i + 1
+
         docs.append(
             Document(
                 page_content=text,
                 metadata={
                     "source": "Prüfungsordnung (PDF)",
-                    "page": i,                  # ZERO-based: Seite = i+1
-                    "pdf_url": PDF_URL,         # Basis-URL
+                    "page": page_num,          # 1-based
+                    "pdf_url": PDF_URL,        # Basis-URL
                 },
             )
         )
     return docs
 
 
+# -------------------------------------------------------------------
+# Loader HTML Hochschulgesetz (từ Storage)
+# -------------------------------------------------------------------
 def load_html_docs():
+    """
+    Hochschulgesetz NRW (giữ 1 Document lớn, chunk sau).
+    Lưu ý:
+    - Ta load bản HTML từ Supabase Storage (trước đó đã crawl/lưu).
+    - get_text(separator="\\n") để giữ cấu trúc tương đối.
+    - Việc chunk sẽ do TextSplitter xử lý.
+    """
     html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
     html = html_bytes.decode("utf-8", errors="ignore")
 
     soup = BeautifulSoup(html, "html.parser")
     text = soup.get_text(separator="\n")
 
-    # HTML nicht in Paragraphen getrennt → wir chunk’en später
     return [
         Document(
             page_content=text,
             metadata={
                 "source": "Hochschulgesetz NRW",
-                # anchor_id wird erst beim Chunken vergeben
+                # anchor_id sẽ được gán sau khi chunk
+                "official_url": OFFICIAL_HG_URL,
             },
         )
     ]
 
 
+# -------------------------------------------------------------------
+# Text-Splitter chung
+# -------------------------------------------------------------------
 def chunk_docs(docs):
+    """
+    Dùng RecursiveCharacterTextSplitter để chia nhỏ nội dung.
+    - chunk_size: 900
+    - chunk_overlap: 100
+    """
     splitter = RecursiveCharacterTextSplitter(
         chunk_size=900,
         chunk_overlap=100,
@@ -64,17 +105,23 @@ def chunk_docs(docs):
     return splitter.split_documents(docs)
 
 
+# -------------------------------------------------------------------
+# Ingest vào Supabase (bảng documents)
+# -------------------------------------------------------------------
 def ingest():
+    # 1) Load nguồn
     pdf_docs = load_pdf_docs()
     hg_docs = load_html_docs()
 
+    # 2) Chunk
     chunks = chunk_docs(pdf_docs + hg_docs)
 
+    # 3) Thêm anchor_id cho từng chunk để nhận diện
     po_idx = 1
     hg_idx = 1
 
     for d in chunks:
-        src = d.metadata["source"]
+        src = d.metadata.get("source")
 
         if src == "Prüfungsordnung (PDF)":
             d.metadata["anchor_id"] = f"po_{po_idx}"
@@ -83,21 +130,25 @@ def ingest():
             d.metadata["anchor_id"] = f"hg_{hg_idx}"
             hg_idx += 1
 
-        # HTML Quelle als vollständige URL
+        # Thêm URL cho HG nếu muốn dùng sau
         if src == "Hochschulgesetz NRW":
-            d.metadata["url"] = f"{HG_URL}#{d.metadata['anchor_id']}"
+            d.metadata["url"] = OFFICIAL_HG_URL
 
+    # 4) Embeddings
     embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
 
     for d in chunks:
         emb = embeddings.embed_query(d.page_content)
-        supabase.table("documents").insert({
-            "content": d.page_content,
-            "metadata": d.metadata,
-            "embedding": emb
-        }).execute()
 
-    print("OK ✔ ingest xong – PDF + HTML mit Quelle-URL")
+        supabase.table("documents").insert(
+            {
+                "content": d.page_content,
+                "metadata": d.metadata,
+                "embedding": emb,
+            }
+        ).execute()
+
+    print("OK ✔ ingest xong – Prüfungsordnung (PDF) + Hochschulgesetz (HTML)")
 
 
 if __name__ == "__main__":

rag_pipeline.py

@@ -1,44 +1,142 @@
 # rag_pipeline.py
 from typing import List, Dict, Any
 from datetime import date
+
 from openai import OpenAI
 from supabase_client import supabase
 from langchain_openai import OpenAIEmbeddings
 
+# -------------------------------------------------------------------
+# OpenAI + Embeddings
+# -------------------------------------------------------------------
 client = OpenAI()
 embedder = OpenAIEmbeddings(model="text-embedding-3-small")
 
+# -------------------------------------------------------------------
+# System Prompt (Rất quan trọng cho độ chính xác)
+# -------------------------------------------------------------------
+SYSTEM_PROMPT = """
+Du bist ein hochpräziser, fachlich korrekter Chatbot für Prüfungsrecht in NRW.
+Du beantwortest ausschließlich auf Grundlage der offiziellen Rechtsquellen:
+
+- Prüfungsordnung (PDF)
+- Hochschulgesetz NRW (recht.nrw.de)
+
+REGELN:
+1. Verwende NUR Informationen aus den bereitgestellten Dokumenten (RAG-Kontext).
+2. Spekuliere nie. Wenn etwas nicht im Dokument steht, sage explizit, dass es dort nicht geregelt ist.
+3. Antworte in klaren, gut strukturierten Sätzen auf Deutsch.
+4. Füge am Ende deiner Antwort keine eigenen Quellen hinzu – die Quellen werden separat im UI angezeigt.
+5. Zitiere sinngemäß, nicht wortwörtlich.
+6. Wenn die Frage unklar ist, bitte freundlich um Präzisierung.
+7. Wenn mehrere Dokumentstellen relevant sind, vergleiche sie kurz.
+
+Wenn du dir unsicher bist, sag offen, dass du es auf Basis der vorliegenden Dokumente nicht sicher beantworten kannst.
+"""
+
 
-def get_relevant_docs(query, k=4):
+# -------------------------------------------------------------------
+# Helper: DB RPC – match_documents
+# -------------------------------------------------------------------
+def get_relevant_docs(query: str, k: int = 4) -> List[Dict[str, Any]]:
+    """
+    Ruft die RPC-Funktion `match_documents` in Supabase auf, um die relevantesten
+    Dokument-Chunks für eine Query zu finden.
+    """
     emb = embedder.embed_query(query)
-    resp = supabase.rpc("match_documents",
-        {"query_embedding": emb, "filter": {}}
+
+    resp = (
+        supabase.rpc(
+            "match_documents",
+            {"query_embedding": emb, "filter": {}},
+        )
+        .execute()
+    )
+
+    data = resp.data or []
+    return data[:k]
+
+
+# -------------------------------------------------------------------
+# Helper: Chat-History in DB speichern
+# -------------------------------------------------------------------
+def save_message(role: str, content: str) -> None:
+    """
+    Speichert eine Chatnachricht (role, content) zusammen mit dem heutigen Datum
+    in der Tabelle `chat_history`.
+    """
+    supabase.table("chat_history").insert(
+        {
+            "session_date": date.today().isoformat(),
+            "role": role,
+            "message": content,
+        }
     ).execute()
-    return (resp.data or [])[:k]
 
 
-def save_message(role, content):
-    supabase.table("chat_history").insert({
-        "session_date": date.today().isoformat(),
-        "role": role,
-        "message": content
-    }).execute()
+# -------------------------------------------------------------------
+# Hauptfunktion: RAG-Antwort generieren
+# -------------------------------------------------------------------
+def rag_answer(query: str, history: Any):
+    """
+    Generiert eine Antwort mit RAG:
 
+    1. Hole relevante Dokumente aus Supabase (Vektorsuche).
+    2. Baue einen kompakten Kontext-String mit Metadaten + Ausschnitten.
+    3. Erzeuge eine Chat-Completion mit SYSTEM_PROMPT + Nutzerfrage + Kontext.
+    4. Speichere User- und Assistant-Nachricht in chat_history.
+    """
 
-def rag_answer(query, history):
+    # 1) Relevante Dokumente
     docs = get_relevant_docs(query)
 
+    # 2) Kontext aus Dokumenten bauen (gekürzt, um "Context Noise" zu vermeiden)
     context = ""
     for i, d in enumerate(docs):
-        meta = d["metadata"]
-        src  = meta.get("source")
+        meta = d.get("metadata", {}) or {}
+        src = meta.get("source", "Unbekannte Quelle")
         page = meta.get("page")
-        page_info = f"(Seite {page+1})" if isinstance(page, int) else ""
-        context += f"[Quelle {i+1}] {src} {page_info}\n{d['content']}\n\n"
+
+        # Seitenangabe (falls vorhanden)
+        if isinstance(page, int):
+            page_info = f"(Seite {page})"
+        else:
+            page_info = ""
+
+        # Text-Ausschnitt
+        snippet = (d.get("content") or "").replace("\n", " ").strip()
+        short = snippet[:450]  # Kontext absichtlich begrenzen
+
+        context += f"[Quelle {i+1}] {src} {page_info}\n{short}\n\n"
+
+    # Optional: kurzen bisherigen Verlauf (für mehr Kontext), nur letzte 6 Einträge
+    history_text = ""
+    if isinstance(history, list):
+        for h in history[-6:]:
+            if isinstance(h, dict):
+                r = h.get("role")
+                c = h.get("content")
+                if r in ("user", "assistant") and c:
+                    history_text += f"{r}: {c}\n"
+
+    # 3) Messages für OpenAI
+    user_prompt = f"""
+Bisheriger Chatverlauf (kurz):
+
+{history_text}
+
+Aktuelle Frage des Nutzers:
+{query}
+
+Relevante Dokumentauszüge:
+{context}
+
+Bitte beantworte die aktuelle Frage ausschließlich auf Basis der Dokumentauszüge.
+"""
 
     messages = [
-        {"role": "system", "content": "Du bist Chatbot für Prüfungsrecht…"},
-        {"role": "user", "content": f"Frage: {query}\n\nDokumente:\n{context}"}
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": user_prompt},
     ]
 
     res = client.chat.completions.create(
@@ -49,6 +147,7 @@ def rag_answer(query, history):
 
     answer = res.choices[0].message.content
 
+    # 4) Verlauf in DB speichern
     save_message("user", query)
     save_message("assistant", answer)
 

supabase_client.py

@@ -2,11 +2,29 @@
 import os
 from supabase import create_client
 
+# -------------------------------------------------------------------
+# Supabase Client (Service-Role, dùng cho đọc/ghi DB + Storage)
+# -------------------------------------------------------------------
 SUPABASE_URL = os.environ["SUPABASE_URL"]
 SUPABASE_SERVICE_ROLE = os.environ["SUPABASE_SERVICE_ROLE"]
 
 supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
 
+
 def load_file_bytes(bucket: str, filename: str) -> bytes:
-    """Tải file từ Supabase Storage nhưng KHÔNG ghi ra local – trả về bytes."""
+    """
+    Tải file từ Supabase Storage mà KHÔNG ghi ra local – trả về bytes.
+
+    Parameters
+    ----------
+    bucket : str
+        Tên bucket trong Supabase Storage.
+    filename : str
+        Đường dẫn/tên file bên trong bucket.
+
+    Returns
+    -------
+    bytes
+        Nội dung file ở dạng bytes.
+    """
     return supabase.storage.from_(bucket).download(filename)