commit

app.py

@@ -1,133 +1,154 @@
-# app.py
+# app.py — UI mit klickbaren Quellen & Voice-Eingabe
 import os
-import base64
 import gradio as gr
 from openai import OpenAI
 
-from supabase_client import load_file_bytes
+from supabase_client import supabase
 from rag_pipeline import rag_answer
 
 client = OpenAI()
 BUCKET = os.environ["SUPABASE_BUCKET"]
 
-# ------------------------------------------
-# Public URLs để mở PDF/HTML khi nhấn Quelle
-# ------------------------------------------
-PDF_URL = f"{os.environ['SUPABASE_URL']}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
-HG_URL  = f"{os.environ['SUPABASE_URL']}/storage/v1/object/public/{BUCKET}/hochschulgesetz.html"
 
+# --------------------------------------------------------
+# Viewer HTML aus Supabase-Dokumenten bauen
+# --------------------------------------------------------
+def build_viewer_html():
+    """Baut HTML-Viewer aus Tabelle documents mit anchor_id."""
+    resp = supabase.table("documents").select("content, metadata").limit(2000).execute()
+    data = resp.data or []
 
-# ------------------------------------------
-# Viewer PDF base64
-# ------------------------------------------
-def encode_pdf_src():
-    pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
-    b64 = base64.b64encode(pdf_bytes).decode("utf-8")
-    return f"data:application/pdf;base64,{b64}"
+    po_blocks = []
+    hg_blocks = []
 
+    for row in data:
+        content = row.get("content") or ""
+        meta = row.get("metadata") or {}
+        src = meta.get("source", "")
+        anchor_id = meta.get("anchor_id")
+        page = meta.get("page", None)
+        page_info = f"(Seite {page})" if page else ""
+
+        block_html = (
+            f"<div id='{anchor_id}' style='margin-bottom: 1rem;'>"
+            f"<b>{src} {page_info}</b><br>{content}</div>"
+        )
+
+        if "Prüfungsordnung" in src:
+            po_blocks.append(block_html)
+        elif "Hochschulgesetz" in src:
+            hg_blocks.append(block_html)
+
+    po_html = "<h3>Prüfungsordnung</h3>" + "".join(po_blocks)
+    hg_html = "<h3>Hochschulgesetz NRW</h3>" + "".join(hg_blocks)
+
+    return po_html, hg_html
 
-# ------------------------------------------
-# HTML viewer
-# ------------------------------------------
-def encode_html():
-    html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
-    return html_bytes.decode("utf-8", errors="ignore")
 
+PO_HTML, HG_HTML = build_viewer_html()
 
-# ------------------------------------------
-# Speech-to-text FIXED
-# ------------------------------------------
-def transcribe(audio_path):
-    if audio_path is None:
+
+# --------------------------------------------------------
+# Speech-to-Text (Whisper, DE)
+# --------------------------------------------------------
+def transcribe(audio_path: str) -> str:
+    if not audio_path:
         return ""
     with open(audio_path, "rb") as f:
         result = client.audio.transcriptions.create(
             model="whisper-1",
             file=f,
-            language="de",      # ép tiếng Đức
-            temperature=0.0     # ổn định kết quả
+            language="de",
+            temperature=0.0,
         )
     return (result.text or "").strip()
 
 
-# ------------------------------------------
-# MAIN CHAT FUNCTION
-# ------------------------------------------
+# --------------------------------------------------------
+# Chat-Funktion
+# --------------------------------------------------------
 def chat_fn(text, audio, history):
     text = (text or "").strip()
 
-    # 1) Ưu tiên text, không dùng audio nếu text có
+    # 1) Priorität: Text. Nur wenn kein Text → Audio
     if text:
         question = text
     elif audio is not None:
         question = transcribe(audio)
     else:
-        return history, "<p>Bitte Text oder Mikrofon benutzen.</p>", None
+        return history, "<p>Bitte Text eingeben oder Mikrofon benutzen.</p>", None
 
     if not question:
-        return history, "<p>Spracherkennung fehlgeschlagen.</p>", None
+        return history, "<p>Spracherkennung fehlgeschlagen. Bitte erneut sprechen.</p>", None
 
-    # 2) RAG
+    # 2) RAG-Antwort
     answer, docs = rag_answer(question, history or [])
 
-    # 3) Build Quellen (click được)
+    # 3) Quellen-HTML mit klickbaren Anchors
     html = "<ol>"
     for i, d in enumerate(docs):
-        meta = d.get("metadata", {}) or {}
+        meta = d.get("metadata") or {}
         src = meta.get("source", "?")
-
-        if "Prüfungsordnung" in src:
-            link = PDF_URL
-        else:
-            link = HG_URL
-
         page = meta.get("page", None)
         page_info = f"(Seite {page})" if page else ""
-
+        anchor_id = meta.get("anchor_id")
         snippet = (d.get("content") or "")[:200]
 
-        html += f"""
-        <li>
-            <a href="{link}" target="_blank">
-                <b>Quelle {i+1}: {src} {page_info}</b>
-            </a><br>
-            {snippet}...
-        </li>
-        """
+        if anchor_id:
+            link = f"#{anchor_id}"
+            html += (
+                f"<li>"
+                f"<a href='{link}'><b>Quelle {i+1}: {src} {page_info}</b></a><br>"
+                f"{snippet}..."
+                f"</li>"
+            )
+        else:
+            html += (
+                f"<li><b>Quelle {i+1}: {src} {page_info}</b><br>"
+                f"{snippet}...</li>"
+            )
     html += "</ol>"
 
-    # 4) Gradio message history
+    # 4) History im messages-Format (für Gradio)
     new_history = (history or []) + [
         {"role": "user", "content": question},
         {"role": "assistant", "content": answer},
     ]
 
-    # Reset audio input
+    # Reset Audio nach dem Senden
     return new_history, html, gr.update(value=None)
 
 
-# ------------------------------------------
-# UI LAYOUT
-# ------------------------------------------
+# --------------------------------------------------------
+# UI Layout
+# --------------------------------------------------------
 with gr.Blocks() as demo:
     gr.Markdown("# ⚖️ Sprachbasierter Chatbot für Prüfungsrecht")
 
     with gr.Row():
         with gr.Column(scale=3):
-            chatbot = gr.Chatbot(label="Chat (RAG)")
-            text_input = gr.Textbox(label="Text Eingabe")
-            audio_input = gr.Audio(type="filepath", label="Spracheingabe (Mikrofon)")
+            chatbot = gr.Chatbot(label="Chat (Prüfungsrecht)")
+            text_input = gr.Textbox(
+                label="Text-Eingabe",
+                placeholder="Frage hier eintippen ..."
+            )
+            audio_input = gr.Audio(
+                type="filepath",
+                label="Spracheingabe (Mikrofon)"
+            )
             send_btn = gr.Button("Senden")
 
         with gr.Column(scale=2):
-            gr.Markdown("### 📄 Prüfungsordnung PDF")
+            gr.Markdown("### 📄 Prüfungsordnung (mit Ankern)")
             gr.HTML(
-                f"<iframe src='{encode_pdf_src()}' width='100%' height='250'></iframe>"
+                f"<div style='overflow:auto; height:250px; "
+                f"border:1px solid #ccc; padding:10px;'>{PO_HTML}</div>"
             )
 
-            gr.Markdown("### 📜 Hochschulgesetz NRW")
+            gr.Markdown("### 📜 Hochschulgesetz NRW (mit Ankern)")
             gr.HTML(
-                f"<div style='overflow:auto;height:250px;'>{encode_html()}</div>"
+                f"<div style='overflow:auto; height:250px; "
+                f"border:1px solid #ccc; padding:10px;'>{HG_HTML}</div>"
             )
 
             sources_html = gr.HTML()

ingest.py

@@ -1,4 +1,4 @@
-# ingest.py
+# ingest.py — Ingest mit anchor_id für jeden Absatz
 import os
 from io import BytesIO
 from bs4 import BeautifulSoup
@@ -6,13 +6,14 @@ from pypdf import PdfReader
 
 from supabase_client import supabase, load_file_bytes
 from langchain_openai import OpenAIEmbeddings
-from langchain_community.vectorstores import SupabaseVectorStore
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 BUCKET = os.environ["SUPABASE_BUCKET"]
 
+
 def load_pdf_docs():
+    """Lädt Prüfungsordnung.pdf aus Supabase (in-memory) und erzeugt pro Seite ein Document."""
     pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
     reader = PdfReader(BytesIO(pdf_bytes))
 
@@ -22,12 +23,17 @@ def load_pdf_docs():
         docs.append(
             Document(
                 page_content=text,
-                metadata={"source": "Prüfungsordnung", "page": i + 1},
+                metadata={
+                    "source": "Prüfungsordnung",
+                    "page": i + 1,
+                },
             )
         )
     return docs
 
+
 def load_html_docs():
+    """Lädt hochschulgesetz.html aus Supabase und extrahiert reinen Text."""
     html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
     html_str = html_bytes.decode("utf-8", errors="ignore")
     soup = BeautifulSoup(html_str, "html.parser")
@@ -40,34 +46,56 @@ def load_html_docs():
         )
     ]
 
+
 def chunk_docs(docs):
+    """Chunking in sinnvolle Absätze."""
     splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000, 
-        chunk_overlap=150
+        chunk_size=800,
+        chunk_overlap=150,
     )
     return splitter.split_documents(docs)
 
-def main():
+
+def ingest():
+    print("📥 Lade Dokumente aus Supabase...")
     pdf_docs = load_pdf_docs()
-    html_docs = load_html_docs()
-    all_docs = pdf_docs + html_docs
+    hg_docs = load_html_docs()
+    all_docs = pdf_docs + hg_docs
 
+    print(f"📄 Rohdokumente geladen: {len(all_docs)}")
     chunks = chunk_docs(all_docs)
+    print(f"✂️  Zu Chunks gesplittet: {len(chunks)}")
 
-    embeddings = OpenAIEmbeddings(
-        model="text-embedding-3-small"
-    )
+    # anchor_id vergeben
+    po_idx = 1
+    hg_idx = 1
+    for d in chunks:
+        src = d.metadata.get("source", "")
+        if "Prüfungsordnung" in src:
+            d.metadata["anchor_id"] = f"po_{po_idx}"
+            po_idx += 1
+        elif "Hochschulgesetz" in src:
+            d.metadata["anchor_id"] = f"hg_{hg_idx}"
+            hg_idx += 1
 
-    SupabaseVectorStore.from_documents(
-        chunks,
-        embeddings,
-        client=supabase,
-        table_name="documents",
-        query_name="match_documents",
-        chunk_size=200,
-    )
+    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+
+    print("🧠 Erzeuge Embeddings & schreibe nach Supabase (Tabelle documents)...")
+    for i, d in enumerate(chunks):
+        emb = embeddings.embed_query(d.page_content)
+        supabase.table("documents").insert(
+            {
+                "content": d.page_content,
+                "metadata": d.metadata,
+                "embedding": emb,
+            }
+        ).execute()
+
+        if (i + 1) % 50 == 0:
+            print(f"   → {i+1}/{len(chunks)} Chunks gespeichert")
+
+    print("✅ Ingest abgeschlossen – Dokumente mit anchor_id in Supabase gespeichert.")
 
-    print("Ingest OK (no local files).")
 
 if __name__ == "__main__":
-    main()
+    ingest()

rag_pipeline.py

@@ -1,4 +1,4 @@
-# rag_pipeline.py
+# rag_pipeline.py — RAG mit Supabase RPC & anchor_id
 import os
 from datetime import date
 from typing import Any, List
@@ -11,28 +11,19 @@ client = OpenAI()
 embedder = OpenAIEmbeddings(model="text-embedding-3-small")
 
 
-# --------------------------------------------------------
-# Lấy tài liệu liên quan từ Supabase bằng RPC match_documents
-# --------------------------------------------------------
-def get_relevant_docs(query: str, k: int = 4) -> List[dict]:
+def get_relevant_docs(query: str, k: int = 6) -> List[dict]:
+    """Ruft match_documents in Supabase auf und liefert die besten k Treffer."""
     embedding = embedder.embed_query(query)
-
     resp = supabase.rpc(
         "match_documents",
-        {
-            "query_embedding": embedding,
-            "filter": {},  # hiện tại không filter thêm
-        },
+        {"query_embedding": embedding, "filter": {}},
     ).execute()
-
     data = resp.data or []
     return data[:k]
 
 
-# --------------------------------------------------------
-# Lưu lịch sử vào bảng chat_history
-# --------------------------------------------------------
 def save_message(role: str, message: str) -> None:
+    """Speichert Nachrichten nach Datum gruppiert in chat_history."""
     today = date.today().isoformat()
     supabase.table("chat_history").insert(
         {
@@ -43,66 +34,59 @@ def save_message(role: str, message: str) -> None:
     ).execute()
 
 
-# --------------------------------------------------------
-# Chuyển history (list tuple / dict) thành text
-# --------------------------------------------------------
 def format_history(history: Any) -> str:
+    """History (list von dict oder tuples) zu einfachem Text für den Prompt."""
     if not history:
         return ""
-    text = ""
+    out = ""
     for turn in history:
-        # tuple / list: (user, assistant)
-        if isinstance(turn, (list, tuple)) and len(turn) >= 2:
-            user_msg = str(turn[0])
-            bot_msg = str(turn[1])
-            text += f"User: {user_msg}\nAssistant: {bot_msg}\n"
-        # dict kiểu {"role": "...", "content": "..."} – phòng khi Gradio thay đổi
-        elif isinstance(turn, dict) and "role" in turn and "content" in turn:
-            role = turn["role"]
-            content = str(turn["content"])
-            if role == "user":
-                text += f"User: {content}\n"
-            elif role == "assistant":
-                text += f"Assistant: {content}\n"
-        # các format khác bỏ qua
-    return text
-
-
-# --------------------------------------------------------
-# Hàm RAG chính
-# --------------------------------------------------------
+        if isinstance(turn, dict) and "role" in turn and "content" in turn:
+            r = turn["role"]
+            c = str(turn["content"])
+            if r == "user":
+                out += f"User: {c}\n"
+            elif r == "assistant":
+                out += f"Assistant: {c}\n"
+        elif isinstance(turn, (list, tuple)) and len(turn) >= 2:
+            out += f"User: {turn[0]}\nAssistant: {turn[1]}\n"
+    return out
+
+
 def rag_answer(question: str, history: Any):
-    # 1) Lấy tài liệu từ vectorstore Supabase
+    """Gibt (Antworttext, Liste von Dokumentdicts) zurück."""
     docs = get_relevant_docs(question)
 
-    # 2) Build context string
+    # Kontext
     context_parts = []
     for i, d in enumerate(docs):
         meta = d.get("metadata") or {}
         src = meta.get("source", "Quelle")
-        page = meta.get("page", None)
-        page_info = f"(Seite {page})" if page is not None else ""
+        page = meta.get("page")
+        page_info = f"(Seite {page})" if page else ""
         text = d.get("content") or ""
-        context_parts.append(f"[Quelle {i+1}] {src} {page_info}\n{text}")
+        context_parts.append(
+            f"[Quelle {i+1}] {src} {page_info}\n{text}"
+        )
     context = "\n\n".join(context_parts) if context_parts else "Keine relevanten Dokumente gefunden."
 
-    # 3) History text
     history_text = format_history(history)
 
-    # 4) System + User prompt
     system_prompt = (
-        "Du bist ein Sprachbasierter Chatbot für Prüfungsrecht an einer Hochschule. "
-        "Du beantwortest Fragen ausschließlich auf Basis der bereitgestellten Dokumente "
+        "Du bist ein spezialisierter Chatbot für Prüfungsrecht an einer Hochschule. "
+        "Du antwortest ausschließlich auf Basis der bereitgestellten Dokumente "
         "(Prüfungsordnung, Hochschulgesetz NRW). "
-        "Wenn die Dokumente keine Antwort liefern, sag ehrlich, dass du es nicht weißt. "
-        "Zitiere relevante Stellen als [Quelle 1], [Quelle 2] usw."
+        "Wenn die Dokumente keine klare Antwort liefern, sag ehrlich, dass es in den vorhandenen Unterlagen nicht eindeutig geregelt ist. "
+        "Zitiere Quellen immer im Format [Quelle X] und nenne, ob sie aus der Prüfungsordnung oder dem Hochschulgesetz stammen."
     )
 
     user_content = (
         f"Frage: {question}\n\n"
         f"Bisheriger Chatverlauf:\n{history_text}\n\n"
         f"Relevante Auszüge aus den Dokumenten:\n{context}\n\n"
-        "Bitte beantworte die Frage präzise und mit Quellenangaben."
+        "Formuliere eine klare, juristisch saubere Antwort. "
+        "Gib am Ende deiner Antwort eine Liste der verwendeten Quellen im Format:\n"
+        "[Quelle 1: Prüfungsordnung, Seite ..., ggf. Paragraph]\n"
+        "[Quelle 2: Hochschulgesetz NRW, Seite ..., ggf. Paragraph]\n"
     )
 
     messages = [
@@ -118,7 +102,6 @@ def rag_answer(question: str, history: Any):
 
     answer = completion.choices[0].message.content
 
-    # 5) Lưu lịch sử vào Supabase
     save_message("user", question)
     save_message("assistant", answer)