commit

app.py

@@ -1,163 +1,78 @@
-# app.py — UI mit klickbaren Quellen & Voice-Eingabe
-import os
+# app.py
 import gradio as gr
+import os
 from openai import OpenAI
-
-from supabase_client import supabase
 from rag_pipeline import rag_answer
+from supabase_client import supabase
 
 client = OpenAI()
-BUCKET = os.environ["SUPABASE_BUCKET"]
-
-
-# --------------------------------------------------------
-# Viewer HTML aus Supabase-Dokumenten bauen
-# --------------------------------------------------------
-def build_viewer_html():
-    """Baut HTML-Viewer aus Tabelle documents mit anchor_id."""
-    resp = supabase.table("documents").select("content, metadata").limit(2000).execute()
-    data = resp.data or []
-
-    po_blocks = []
-    hg_blocks = []
-
-    for row in data:
-        content = row.get("content") or ""
-        meta = row.get("metadata") or {}
-        src = meta.get("source", "")
-        anchor_id = meta.get("anchor_id")
-        page = meta.get("page", None)
-        page_info = f"(Seite {page})" if page else ""
-
-        block_html = (
-            f"<div id='{anchor_id}' style='margin-bottom: 1rem;'>"
-            f"<b>{src} {page_info}</b><br>{content}</div>"
-        )
 
+def build_viewer():
+    resp = supabase.table("documents").select("content, metadata").execute()
+    items = resp.data or []
+    po_html = []
+    hg_html = []
+    for row in items:
+        meta = row["metadata"]
+        src = meta["source"]
+        anchor = meta["anchor_id"]
+        page = meta.get("page", "")
+        block_html = f"<div id='{anchor}'><b>{src} {page}</b><br>{row['content']}</div>"
         if "Prüfungsordnung" in src:
-            po_blocks.append(block_html)
-        elif "Hochschulgesetz" in src:
-            hg_blocks.append(block_html)
-
-    po_html = "<h3>Prüfungsordnung</h3>" + "".join(po_blocks)
-    hg_html = "<h3>Hochschulgesetz NRW</h3>" + "".join(hg_blocks)
-
-    return po_html, hg_html
-
-
-PO_HTML, HG_HTML = build_viewer_html()
+            po_html.append(block_html)
+        else:
+            hg_html.append(block_html)
+    return "".join(po_html), "".join(hg_html)
 
+PO_HTML, HG_HTML = build_viewer()
 
-# --------------------------------------------------------
-# Speech-to-Text (Whisper, DE)
-# --------------------------------------------------------
-def transcribe(audio_path: str) -> str:
-    if not audio_path:
+def transcribe(audio):
+    if audio is None:
         return ""
-    with open(audio_path, "rb") as f:
-        result = client.audio.transcriptions.create(
-            model="whisper-1",
-            file=f,
-            language="de",
-            temperature=0.0,
+    with open(audio, "rb") as f:
+        res = client.audio.transcriptions.create(
+            model="whisper-1", file=f, language="de", temperature=0
         )
-    return (result.text or "").strip()
+    return res.text.strip()
 
-
-# --------------------------------------------------------
-# Chat-Funktion
-# --------------------------------------------------------
 def chat_fn(text, audio, history):
-    text = (text or "").strip()
-
-    # 1) Priorität: Text. Nur wenn kein Text → Audio
+    text = text.strip() if text else ""
     if text:
-        question = text
-    elif audio is not None:
-        question = transcribe(audio)
+        q = text
     else:
-        return history, "<p>Bitte Text eingeben oder Mikrofon benutzen.</p>", None
-
-    if not question:
-        return history, "<p>Spracherkennung fehlgeschlagen. Bitte erneut sprechen.</p>", None
+        q = transcribe(audio)
 
-    # 2) RAG-Antwort
-    answer, docs = rag_answer(question, history or [])
+    answer, docs = rag_answer(q, history or [])
 
-    # 3) Quellen-HTML mit klickbaren Anchors
     html = "<ol>"
     for i, d in enumerate(docs):
-        meta = d.get("metadata") or {}
-        src = meta.get("source", "?")
-        page = meta.get("page", None)
-        page_info = f"(Seite {page})" if page else ""
-        anchor_id = meta.get("anchor_id")
-        snippet = (d.get("content") or "")[:200]
-
-        if anchor_id:
-            link = f"#{anchor_id}"
-            html += (
-                f"<li>"
-                f"<a href='{link}'><b>Quelle {i+1}: {src} {page_info}</b></a><br>"
-                f"{snippet}..."
-                f"</li>"
-            )
-        else:
-            html += (
-                f"<li><b>Quelle {i+1}: {src} {page_info}</b><br>"
-                f"{snippet}...</li>"
-            )
+        meta = d["metadata"]
+        anchor = meta["anchor_id"]
+        snippet = d["content"][:200]
+        html += f"<li><a href='#{anchor}'><b>Quelle {i+1}</b></a><br>{snippet}...</li>"
     html += "</ol>"
 
-    # 4) History im messages-Format (für Gradio)
-    new_history = (history or []) + [
-        {"role": "user", "content": question},
-        {"role": "assistant", "content": answer},
+    new_hist = (history or []) + [
+        {"role": "user", "content": q},
+        {"role": "assistant", "content": answer}
     ]
 
-    # Reset Audio nach dem Senden
-    return new_history, html, gr.update(value=None)
-
+    return new_hist, html, gr.update(value=None)  # reset audio
 
-# --------------------------------------------------------
-# UI Layout
-# --------------------------------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("# ⚖️ Sprachbasierter Chatbot für Prüfungsrecht")
-
-    with gr.Row():
-        with gr.Column(scale=3):
-            chatbot = gr.Chatbot(label="Chat (Prüfungsrecht)")
-            text_input = gr.Textbox(
-                label="Text-Eingabe",
-                placeholder="Frage hier eintippen ..."
-            )
-            audio_input = gr.Audio(
-                type="filepath",
-                label="Spracheingabe (Mikrofon)"
-            )
-            send_btn = gr.Button("Senden")
-
-        with gr.Column(scale=2):
-            gr.Markdown("### 📄 Prüfungsordnung (mit Ankern)")
-            gr.HTML(
-                f"<div style='overflow:auto; height:250px; "
-                f"border:1px solid #ccc; padding:10px;'>{PO_HTML}</div>"
-            )
-
-            gr.Markdown("### 📜 Hochschulgesetz NRW (mit Ankern)")
-            gr.HTML(
-                f"<div style='overflow:auto; height:250px; "
-                f"border:1px solid #ccc; padding:10px;'>{HG_HTML}</div>"
-            )
+    chatbot = gr.Chatbot()
+    text_input = gr.Textbox(label="Text Eingabe")
+    audio_input = gr.Audio(type="filepath", label="Mikrofon")
+    send = gr.Button("Senden")
 
-            sources_html = gr.HTML()
+    po_view = gr.HTML(f"<div style='height:250px; overflow:auto'>{PO_HTML}</div>")
+    hg_view = gr.HTML(f"<div style='height:250px; overflow:auto'>{HG_HTML}</div>")
+    sources = gr.HTML()
 
-    send_btn.click(
+    send.click(
         chat_fn,
         inputs=[text_input, audio_input, chatbot],
-        outputs=[chatbot, sources_html, audio_input],
+        outputs=[chatbot, sources, audio_input]
     )
 
-if __name__ == "__main__":
-    demo.launch(ssr_mode=False)
+demo.launch()

ingest.py

@@ -1,4 +1,4 @@
-# ingest.py — Ingest mit anchor_id für jeden Absatz
+# ingest.py
 import os
 from io import BytesIO
 from bs4 import BeautifulSoup
@@ -6,96 +6,68 @@ from pypdf import PdfReader
 
 from supabase_client import supabase, load_file_bytes
 from langchain_openai import OpenAIEmbeddings
-from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
 
 BUCKET = os.environ["SUPABASE_BUCKET"]
 
-
 def load_pdf_docs():
-    """Lädt Prüfungsordnung.pdf aus Supabase (in-memory) und erzeugt pro Seite ein Document."""
     pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
     reader = PdfReader(BytesIO(pdf_bytes))
-
     docs = []
     for i, page in enumerate(reader.pages):
         text = page.extract_text() or ""
-        docs.append(
-            Document(
-                page_content=text,
-                metadata={
-                    "source": "Prüfungsordnung",
-                    "page": i + 1,
-                },
-            )
-        )
+        docs.append(Document(
+            page_content=text,
+            metadata={"source": "Prüfungsordnung", "page": i + 1},
+        ))
     return docs
 
-
 def load_html_docs():
-    """Lädt hochschulgesetz.html aus Supabase und extrahiert reinen Text."""
     html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
-    html_str = html_bytes.decode("utf-8", errors="ignore")
-    soup = BeautifulSoup(html_str, "html.parser")
+    html = html_bytes.decode("utf-8", errors="ignore")
+    soup = BeautifulSoup(html, "html.parser")
     text = soup.get_text(separator="\n")
-
-    return [
-        Document(
-            page_content=text,
-            metadata={"source": "Hochschulgesetz NRW"},
-        )
-    ]
-
+    return [Document(
+        page_content=text,
+        metadata={"source": "Hochschulgesetz NRW"},
+    )]
 
 def chunk_docs(docs):
-    """Chunking in sinnvolle Absätze."""
     splitter = RecursiveCharacterTextSplitter(
-        chunk_size=800,
-        chunk_overlap=150,
-    )
+        chunk_size=900, chunk_overlap=100)
     return splitter.split_documents(docs)
 
-
 def ingest():
-    print("📥 Lade Dokumente aus Supabase...")
     pdf_docs = load_pdf_docs()
     hg_docs = load_html_docs()
-    all_docs = pdf_docs + hg_docs
+    chunks = chunk_docs(pdf_docs + hg_docs)
 
-    print(f"📄 Rohdokumente geladen: {len(all_docs)}")
-    chunks = chunk_docs(all_docs)
-    print(f"✂️  Zu Chunks gesplittet: {len(chunks)}")
+    # gán anchor_id
+    po_index = 1
+    hg_index = 1
 
-    # anchor_id vergeben
-    po_idx = 1
-    hg_idx = 1
     for d in chunks:
-        src = d.metadata.get("source", "")
-        if "Prüfungsordnung" in src:
-            d.metadata["anchor_id"] = f"po_{po_idx}"
-            po_idx += 1
-        elif "Hochschulgesetz" in src:
-            d.metadata["anchor_id"] = f"hg_{hg_idx}"
-            hg_idx += 1
+        src = d.metadata["source"]
+        if src == "Prüfungsordnung":
+            d.metadata["anchor_id"] = f"po_{po_index}"
+            po_index += 1
+        else:
+            d.metadata["anchor_id"] = f"hg_{hg_index}"
+            hg_index += 1
 
     embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
 
-    print("🧠 Erzeuge Embeddings & schreibe nach Supabase (Tabelle documents)...")
-    for i, d in enumerate(chunks):
+    # insert thủ công
+    for d in chunks:
         emb = embeddings.embed_query(d.page_content)
-        supabase.table("documents").insert(
-            {
-                "content": d.page_content,
-                "metadata": d.metadata,
-                "embedding": emb,
-            }
-        ).execute()
-
-        if (i + 1) % 50 == 0:
-            print(f"   → {i+1}/{len(chunks)} Chunks gespeichert")
-
-    print("✅ Ingest abgeschlossen – Dokumente mit anchor_id in Supabase gespeichert.")
+        supabase.table("documents").insert({
+            "content": d.page_content,
+            "metadata": d.metadata,
+            "embedding": emb
+        }).execute()
 
+    print("OK ✔ ingest xong – đã có anchor_id cho tất cả documents")
 
 if __name__ == "__main__":
     ingest()

rag_pipeline.py

@@ -1,108 +1,51 @@
-# rag_pipeline.py — RAG mit Supabase RPC & anchor_id
+# rag_pipeline.py
 import os
 from datetime import date
-from typing import Any, List
-
 from openai import OpenAI
-from langchain_openai import OpenAIEmbeddings
 from supabase_client import supabase
+from langchain_openai import OpenAIEmbeddings
 
 client = OpenAI()
 embedder = OpenAIEmbeddings(model="text-embedding-3-small")
 
-
-def get_relevant_docs(query: str, k: int = 6) -> List[dict]:
-    """Ruft match_documents in Supabase auf und liefert die besten k Treffer."""
-    embedding = embedder.embed_query(query)
-    resp = supabase.rpc(
-        "match_documents",
-        {"query_embedding": embedding, "filter": {}},
-    ).execute()
-    data = resp.data or []
-    return data[:k]
-
-
-def save_message(role: str, message: str) -> None:
-    """Speichert Nachrichten nach Datum gruppiert in chat_history."""
-    today = date.today().isoformat()
-    supabase.table("chat_history").insert(
-        {
-            "session_date": today,
-            "role": role,
-            "message": message,
-        }
-    ).execute()
-
-
-def format_history(history: Any) -> str:
-    """History (list von dict oder tuples) zu einfachem Text für den Prompt."""
-    if not history:
-        return ""
-    out = ""
-    for turn in history:
-        if isinstance(turn, dict) and "role" in turn and "content" in turn:
-            r = turn["role"]
-            c = str(turn["content"])
-            if r == "user":
-                out += f"User: {c}\n"
-            elif r == "assistant":
-                out += f"Assistant: {c}\n"
-        elif isinstance(turn, (list, tuple)) and len(turn) >= 2:
-            out += f"User: {turn[0]}\nAssistant: {turn[1]}\n"
-    return out
-
-
-def rag_answer(question: str, history: Any):
-    """Gibt (Antworttext, Liste von Dokumentdicts) zurück."""
-    docs = get_relevant_docs(question)
-
-    # Kontext
-    context_parts = []
+def get_relevant_docs(query, k=4):
+    emb = embedder.embed_query(query)
+    resp = supabase.rpc("match_documents", {
+        "query_embedding": emb,
+        "filter": {}
+    }).execute()
+    return (resp.data or [])[:k]
+
+def save_message(role, content):
+    supabase.table("chat_history").insert({
+        "session_date": date.today().isoformat(),
+        "role": role,
+        "message": content
+    }).execute()
+
+def rag_answer(query, history):
+    docs = get_relevant_docs(query)
+    context = ""
     for i, d in enumerate(docs):
-        meta = d.get("metadata") or {}
-        src = meta.get("source", "Quelle")
+        meta = d["metadata"]
+        src = meta["source"]
         page = meta.get("page")
         page_info = f"(Seite {page})" if page else ""
-        text = d.get("content") or ""
-        context_parts.append(
-            f"[Quelle {i+1}] {src} {page_info}\n{text}"
-        )
-    context = "\n\n".join(context_parts) if context_parts else "Keine relevanten Dokumente gefunden."
-
-    history_text = format_history(history)
-
-    system_prompt = (
-        "Du bist ein spezialisierter Chatbot für Prüfungsrecht an einer Hochschule. "
-        "Du antwortest ausschließlich auf Basis der bereitgestellten Dokumente "
-        "(Prüfungsordnung, Hochschulgesetz NRW). "
-        "Wenn die Dokumente keine klare Antwort liefern, sag ehrlich, dass es in den vorhandenen Unterlagen nicht eindeutig geregelt ist. "
-        "Zitiere Quellen immer im Format [Quelle X] und nenne, ob sie aus der Prüfungsordnung oder dem Hochschulgesetz stammen."
-    )
-
-    user_content = (
-        f"Frage: {question}\n\n"
-        f"Bisheriger Chatverlauf:\n{history_text}\n\n"
-        f"Relevante Auszüge aus den Dokumenten:\n{context}\n\n"
-        "Formuliere eine klare, juristisch saubere Antwort. "
-        "Gib am Ende deiner Antwort eine Liste der verwendeten Quellen im Format:\n"
-        "[Quelle 1: Prüfungsordnung, Seite ..., ggf. Paragraph]\n"
-        "[Quelle 2: Hochschulgesetz NRW, Seite ..., ggf. Paragraph]\n"
-    )
+        context += f"[Quelle {i+1}] {src} {page_info}\n{d['content']}\n\n"
 
     messages = [
-        {"role": "system", "content": system_prompt},
-        {"role": "user", "content": user_content},
+        {"role": "system", "content": "Du bist Chatbot für Prüfungsrecht…"},
+        {"role": "user", "content": f"Frage: {query}\n\nDokumente:\n{context}"}
     ]
 
-    completion = client.chat.completions.create(
+    res = client.chat.completions.create(
         model="gpt-4.1-mini",
         messages=messages,
-        temperature=0.1,
+        temperature=0
     )
 
-    answer = completion.choices[0].message.content
-
-    save_message("user", question)
+    answer = res.choices[0].message.content
+    save_message("user", query)
     save_message("assistant", answer)
 
     return answer, docs