commit

app.py

@@ -1,114 +1,201 @@
-# app.py
-import os
-import re
-import base64
-import io
-import soundfile as sf
+# app.py – Prüfungsrechts-Chatbot mit OpenAI (Supabase RAG)
 
 import gradio as gr
-from openai import OpenAI
 
-from supabase_client import load_file_bytes
-from rag_pipeline import rag_answer
+from load_documents import load_documents, PDF_URL, HG_HTML_URL
+from split_documents import split_documents
+from vectorstore import build_vectorstore
+from retriever import get_retriever
+from llm import load_llm
+from rag_pipeline import answer
+from speech_io import transcribe_audio, synthesize_speech
 
+# =====================================================
+# INITIALISIERUNG (beim Start der Space einmalig)
+# =====================================================
 
-client = OpenAI()
+print("🔹 Lade Dokumente aus Supabase …")
+_docs = load_documents()
 
-BUCKET = os.environ["SUPABASE_BUCKET"]
-SUPABASE_URL = os.environ["SUPABASE_URL"]
+print("🔹 Splitte Dokumente …")
+_chunks = split_documents(_docs)
 
-PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
-HG_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
+print("🔹 Baue VectorStore …")
+_vs = build_vectorstore(_chunks)
 
+print("🔹 Erzeuge Retriever …")
+_retriever = get_retriever(_vs)
 
-def encode_pdf_src():
-    b = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
-    return f"data:application/pdf;base64,{base64.b64encode(b).decode()}"
+print("🔹 Lade OpenAI LLM …")
+_llm = load_llm()
 
+# =====================================================
+# Quellen formatieren – Markdown im Chat
+# =====================================================
 
-# Whisper cleanup
-def clean_text(t):
-    t = t.lower()
-    t = re.sub(r"[^\wäöüß ,.?-]+", " ", t)
-    return t.strip().capitalize()
+def format_sources_markdown(sources):
+    if not sources:
+        return ""
 
+    lines = ["", "### 📚 Quellen (verwendete Dokumentstellen):"]
 
-def transcribe(audio):
-    if audio is None:
-        return ""
-    audio_data, sr = audio
-    buf = io.BytesIO()
-    sf.write(buf, audio_data, sr, format="WAV")
-    buf.seek(0)
+    for s in sources:
+        sid = s["id"]
+        src = s["source"]
+        page = s["page"]
+        url = s["url"]
+        snippet = s["snippet"]
+
+        if page:
+            title = f"Quelle {sid} – {src}, Seite {page}"
+        else:
+            title = f"Quelle {sid} – {src}"
+
+        if url:
+            base = f"- [{title}]({url})"
+        else:
+            base = f"- {title}"
+
+        lines.append(base)
+        if snippet:
+            lines.append(f"  > {snippet}")
+
+    return "\n".join(lines)
 
-    result = client.audio.transcriptions.create(
-        model="whisper-1", file=buf, filename="audio.wav", language="de"
+# =====================================================
+# TEXT CHATBOT
+# =====================================================
+
+def chatbot_text(user_message, history):
+    if not user_message:
+        return history, ""
+
+    answer_text, sources = answer(
+        question=user_message,
+        retriever=_retriever,
+        chat_model=_llm,
     )
-    return clean_text(result.text or "")
 
+    quellen_block = format_sources_markdown(sources)
+    bot_msg = answer_text + "\n\n" + quellen_block
 
-def chat_fn(mode, text, audio, history):
-    history = history or []
+    history = history + [
+        {"role": "user", "content": user_message},
+        {"role": "assistant", "content": bot_msg},
+    ]
 
-    if mode == "text":
-        q = text.strip()
-    else:
-        q = transcribe(audio)
+    return history, ""
 
-    if not q:
-        return history, "Keine gültige Eingabe erkannt.", None
+# =====================================================
+# VOICE CHATBOT
+# =====================================================
 
-    answer, docs = rag_answer(q, history)
+def chatbot_voice(audio_path, history):
+    text = transcribe_audio(audio_path)
+    if not text:
+        return history, None, ""
 
-    quellen = ["", "### 📚 Verwendete Quellen"]
-    for i, d in enumerate(docs):
-        src = d["source"]
-        pg = d["page"]
+    history = history + [{"role": "user", "content": text}]
 
-        if src.startswith("Prüfungsordnung"):
-            url = f"{PDF_URL}#page={pg}"
-        else:
-            url = HG_URL
+    answer_text, sources = answer(
+        question=text,
+        retriever=_retriever,
+        chat_model=_llm,
+    )
+
+    quellen_block = format_sources_markdown(sources)
+    bot_msg = answer_text + "\n\n" + quellen_block
+
+    history = history + [{"role": "assistant", "content": bot_msg}]
+
+    audio = synthesize_speech(bot_msg)
+
+    return history, audio, ""
 
-        snippet = d["snippet"][:200]
-        quellen.append(f"- **{src}** (Seite {pg}) → [{url}]({url})  \n  „{snippet}…”")
+# =====================================================
+# Wieder-Vorlesen der letzten Antwort
+# =====================================================
 
-    bot = answer + "\n\n" + "\n".join(quellen)
+def read_last_answer(history):
+    if not history:
+        return None
 
-    return history + [
-        {"role": "user", "content": q},
-        {"role": "assistant", "content": bot},
-    ], bot, gr.update(value=None)
+    for msg in reversed(history):
+        if msg["role"] == "assistant":
+            return synthesize_speech(msg["content"])
+    return None
 
+# =====================================================
+# UI (Gradio)
+# =====================================================
 
-with gr.Blocks() as demo:
-    gr.Markdown("# ⚖️ Prüfungsrechts-Assistent NRW")
+with gr.Blocks(title="Prüfungsrechts-Chatbot (Supabase + OpenAI)") as demo:
+
+    gr.Markdown("# 🧑‍⚖️ Prüfungsrechts-Chatbot (Supabase RAG + OpenAI)")
+    gr.Markdown(
+        "Fragen zum Prüfungsrecht (Prüfungsordnung + Hochschulgesetz NRW). "
+        "Antworten mit Quellenangabe und Direktlinks."
+    )
 
     with gr.Row():
-        with gr.Column(scale=3):
-            chatbot = gr.Chatbot()
 
-            mode = gr.Radio(["text", "audio"], value="text", label="Eingabemodus")
-            text = gr.Textbox(label="Text eingeben")
+        # ---------- LINKER BEREICH: CHAT ----------
+        with gr.Column(scale=2):
+
+            chatbot = gr.Chatbot(
+                type="messages",
+                label="Chat",
+                height=550,
+            )
 
-            audio = gr.Audio(
-                sources=["microphone"],
-                type="numpy",
-                format="wav",
-                label="Spracheingabe (Mikrofon)",
+            msg = gr.Textbox(
+                label="Frage eingeben",
+                placeholder="Stelle deine Frage zum Prüfungsrecht …",
+                autofocus=True,
             )
+            msg.submit(chatbot_text, [msg, chatbot], [chatbot, msg])
 
-            send = gr.Button("Senden")
-            preview = gr.Markdown()
+            send_btn = gr.Button("Senden (Text)")
+            send_btn.click(chatbot_text, [msg, chatbot], [chatbot, msg])
 
-        with gr.Column(scale=2):
-            gr.Markdown("### 📄 Prüfungsordnung (PDF)")
-            gr.HTML(f"<iframe src='{encode_pdf_src()}' width='100%' height='260'></iframe>")
+            gr.Markdown("### 🎙️ Spracheingabe")
+
+            voice_in = gr.Audio(sources=["microphone"], type="filepath")
+            voice_out = gr.Audio(label="Vorgelesene Antwort", type="numpy")
 
-            gr.Markdown("### 📘 Hochschulgesetz NRW")
-            gr.HTML(f"<iframe src='{HG_URL}' width='100%' height='260'></iframe>")
+            send_voice_btn = gr.Button("Sprechen & Senden")
+            send_voice_btn.click(
+                chatbot_voice,
+                [voice_in, chatbot],
+                [chatbot, voice_out, msg],
+            )
+
+            read_btn = gr.Button("Antwort erneut vorlesen")
+            read_btn.click(read_last_answer, [chatbot], [voice_out])
+
+            clear_btn = gr.Button("Chat löschen")
+            clear_btn.click(lambda: [], None, chatbot)
+
+        # ---------- RECHTER BEREICH: VIEWER ----------
+        with gr.Column(scale=1):
 
-    send.click(chat_fn, [mode, text, audio, chatbot], [chatbot, preview, audio])
+            gr.Markdown("### 📄 Prüfungsordnung (PDF)")
+            gr.HTML(
+                f"""
+                <iframe src="{PDF_URL}"
+                        style="width:100%; height:330px; border:none;">
+                </iframe>
+                """
+            )
+
+            gr.Markdown("### 📘 Hochschulgesetz NRW (Paragraph-Viewer)")
+            gr.HTML(
+                f"""
+                <iframe src="{HG_HTML_URL}"
+                        style="width:100%; height:330px; border:none;">
+                </iframe>
+                """
+            )
 
 if __name__ == "__main__":
-    demo.queue().launch()
+    demo.queue().launch(ssr_mode=False, show_error=True)

build_hg_viewer.py

@@ -0,0 +1,313 @@
+# build_hg_viewer.py
+import os
+from supabase import create_client
+from dotenv import load_dotenv
+
+load_dotenv()
+
+SUPABASE_URL = os.environ["SUPABASE_URL"]
+SUPABASE_SERVICE_ROLE = os.environ["SUPABASE_SERVICE_ROLE"]
+
+if not SUPABASE_URL or not SUPABASE_SERVICE_ROLE:
+    raise RuntimeError("Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE")
+
+supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
+
+from upload_weblink_to_supabase import extract_paragraphs
+
+# ======== HTML TEMPLATE ========
+VIEW_TEMPLATE = """
+<!DOCTYPE html>
+<html lang="de">
+<head>
+<meta charset="UTF-8">
+<title>Hochschulgesetz NRW – Paragraph Viewer</title>
+
+<style>
+body {
+    font-family: Arial, sans-serif;
+    margin: 0;
+    padding: 0;
+    display: flex;
+}
+
+/* ----------- SIDEBAR ------------- */
+#sidebar {
+    width: 280px;
+    height: 100vh;
+    overflow-y: auto;
+    background: #f5f5f5;
+    border-right: 1px solid #ccc;
+    padding: 15px;
+    position: sticky;
+    top: 0;
+}
+
+#sidebar h2 {
+    margin-top: 0;
+}
+
+#searchBox {
+    width: 100%;
+    padding: 8px;
+    font-size: 15px;
+    margin-bottom: 10px;
+    border: 1px solid #aaa;
+    border-radius: 5px;
+}
+
+.sidebar-link {
+    display: block;
+    padding: 6px 8px;
+    margin-bottom: 4px;
+    text-decoration: none;
+    color: #003366;
+    border-radius: 4px;
+}
+
+.sidebar-link:hover {
+    background: #e0e7ff;
+    color: #001d4d;
+}
+
+/* ----------- CONTENT ------------- */
+#content {
+    flex: 1;
+    padding: 25px;
+    max-width: 900px;
+}
+
+/* Absatz block */
+.para {
+    padding: 20px 0;
+    border-bottom: 1px solid #ddd;
+}
+
+.para h2 {
+    color: #003366;
+    margin-bottom: 10px;
+}
+
+/* ----------- Fußnoten ------------- */
+.fn-block {
+    background: #fafafa;
+    border-left: 4px solid #999;
+    padding: 12px;
+    margin-top: 10px;
+    margin-bottom: 25px;
+}
+
+.fn-toggle {
+    cursor: pointer;
+    font-weight: bold;
+    color: #003366;
+    margin-bottom: 5px;
+}
+
+.fn-content {
+    display: none;
+    padding-left: 10px;
+}
+
+.fn-title {
+    font-weight: bold;
+    margin-bottom: 6px;
+}
+
+.fn-item {
+    margin-bottom: 8px;
+}
+
+/* ----------- Highlight beim Öffnen ------------- */
+.highlight {
+    animation: flash 2s ease-in-out;
+    background: #fff8c6 !important;
+}
+
+@keyframes flash {
+    0% { background: #fff8c6; }
+    100% { background: transparent; }
+}
+
+/* Keyword highlight */
+.keyword {
+    background: yellow;
+    padding: 2px 3px;
+    border-radius: 3px;
+}
+
+/* Back to top button */
+#topBtn {
+    position: fixed;
+    bottom: 25px;
+    right: 25px;
+    background: #003366;
+    color: white;
+    border-radius: 8px;
+    padding: 10px 14px;
+    cursor: pointer;
+    font-size: 16px;
+    display: none;
+}
+</style>
+
+</head>
+<body>
+
+<div id="sidebar">
+    <h2>Inhaltsverzeichnis</h2>
+    <input type="text" id="searchBox" placeholder="Suchen nach § …">
+    <!-- SIDEBAR_LINKS -->
+</div>
+
+<div id="content">
+    <h1>Hochschulgesetz NRW – Paragraph Viewer</h1>
+    <!-- PARAGRAPH_CONTENT -->
+</div>
+
+<div id="topBtn" onclick="scrollToTop()">⬆️ Top</div>
+
+<script>
+// ------ TỰ ĐỘNG HIGHLIGHT Absatz khi có #anchor HIGHLIGHT ABSATZ & SCROLL ------
+window.onload = function() {
+    const anchor = window.location.hash.substring(1);
+    const params = new URLSearchParams(window.location.search);
+    const keywords = params.get("k");
+
+    if (anchor) {
+        const el = document.getElementById(anchor);
+        if (el) {
+            el.classList.add("highlight");
+            el.scrollIntoView({ behavior: "smooth", block: "center" });
+        }
+    }
+
+    /* KEYWORD HIGHLIGHT */
+    if (keywords) {
+        const words = keywords.split("%20");
+        highlightKeywords(words);
+    }
+};
+
+/* --- KEYWORD HIGHLIGHT FUNCTION --- */
+function highlightKeywords(words) {
+    const container = document.getElementById("content");
+    let html = container.innerHTML;
+
+    words.forEach(word => {
+        if (word.length < 2) return;
+        const regex = new RegExp(`(${decodeURIComponent(word)})`, "gi");
+        html = html.replace(regex, `<span class="keyword">$1</span>`);
+    });
+
+    container.innerHTML = html;
+}
+
+/* --- SEARCH IN SIDEBAR --- */
+document.getElementById("searchBox").addEventListener("input", function() {
+    const q = this.value.toLowerCase();
+    document.querySelectorAll(".sidebar-link").forEach(link => {
+        const txt = link.innerText.toLowerCase();
+        link.style.display = txt.includes(q) ? "block" : "none";
+    });
+});
+
+/* --- COLLAPSIBLE FUSSNOTEN --- */
+document.addEventListener("click", function(e) {
+    if (e.target.classList.contains("fn-toggle")) {
+        const content = e.target.nextElementSibling;
+        content.style.display = content.style.display === "block" ? "none" : "block";
+    }
+});
+
+/* --- BACK TO TOP BUTTON --- */
+window.onscroll = function() {
+    document.getElementById("topBtn").style.display =
+        window.scrollY > 300 ? "block" : "none";
+};
+
+function scrollToTop() {
+    window.scrollTo({ top: 0, behavior: 'smooth' });
+}
+
+</script>
+
+</body>
+</html>
+"""
+
+# -------------------------------------------------------------------
+# 2. BUILD VIEWER
+# -------------------------------------------------------------------
+
+def build_html():
+    print(">>> Lade Paragraphs aus Supabase...")
+    paras = extract_paragraphs()
+
+    sidebar_links = ""
+    content_html = ""
+
+    for p in paras:
+        pid = p["abs_id"]
+        title = p["title"]
+        body = p["content"]
+
+        # Sidebar item
+        sidebar_links += f'<a class="sidebar-link" href="#{pid}">{title}</a>\n'
+
+        # Fußnoten tách riêng (bắt đầu bằng "Fn 1", "Fn 2", ...)
+        lines = body.split("\n")
+        main_text = []
+        fn_text = []
+        in_fn = False
+
+        for line in lines:
+            if line.startswith("Fn "):
+                in_fn = True
+            if in_fn:
+                fn_text.append(line)
+            else:
+                main_text.append(line)
+
+        footnotes_html = ""
+        if fn_text:
+            footnotes_html += '<div class="fn-block">'
+            footnotes_html += '<div class="fn-title">Fußnoten:</div>'
+            for fn in fn_text:
+                footnotes_html += f'<div class="fn-item">{fn}</div>'
+            footnotes_html += "</div>"
+
+        # Paragraph block
+        content_html += f"""
+<div class="para" id="{pid}">
+    <h2>{title}</h2>
+    <div>{'<br>'.join(main_text)}</div>
+    {footnotes_html}
+</div>
+"""
+
+    html = VIEW_TEMPLATE.replace("<!-- SIDEBAR_LINKS -->", sidebar_links)
+    html = html.replace("<!-- PARAGRAPH_CONTENT -->", content_html)
+
+    return html
+
+# -------------------------------------------------------------------
+# 3. UPLOAD TO SUPABASE STORAGE
+# -------------------------------------------------------------------
+
+def upload_html():
+    html = build_html()
+
+    supabase.storage.from_("hg_viewer").update(
+        "hg_clean.html",
+        html.encode("utf-8"),
+        {
+            "content-type": "text/html",
+            "x-upsert": "true"
+        }
+    )
+
+    print("✔ hg_clean.html uploaded!")
+
+if __name__ == "__main__":
+    upload_html()

embeddings.py

@@ -0,0 +1,24 @@
+# embeddings.py – OpenAI Version (text-embedding-3-small)
+
+import os
+from langchain_openai import OpenAIEmbeddings
+
+EMBED_MODEL = "text-embedding-3-small"
+
+def get_embeddings():
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY fehlt. Bitte als Secret im HuggingFace Space setzen."
+        )
+
+    print(f">>> Lade OpenAI Embedding Model: {EMBED_MODEL}")
+    emb = OpenAIEmbeddings(
+        model=EMBED_MODEL,
+        api_key=api_key,
+    )
+    return emb
+
+if __name__ == "__main__":
+    e = get_embeddings()
+    print(e.embed_query("Test"))

ingest.py

@@ -1,94 +0,0 @@
-# ingest.py
-import os
-from io import BytesIO
-from bs4 import BeautifulSoup
-from pypdf import PdfReader
-
-from supabase_client import supabase, load_file_bytes
-from langchain_openai import OpenAIEmbeddings
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_core.documents import Document
-
-BUCKET = os.environ["SUPABASE_BUCKET"]
-SUPABASE_URL = os.environ["SUPABASE_URL"]
-
-PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
-OFFICIAL_HG_URL = (
-    "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
-)
-
-
-# ---------------- Loaders ----------------
-def load_pdf_docs():
-    pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
-    reader = PdfReader(BytesIO(pdf_bytes))
-
-    docs = []
-    for i, p in enumerate(reader.pages):
-        text = p.extract_text() or ""
-        docs.append(
-            Document(
-                page_content=text,
-                metadata={
-                    "source": "Prüfungsordnung (PDF)",
-                    "page": i + 1,
-                    "pdf_url": PDF_URL,
-                },
-            )
-        )
-    return docs
-
-
-def load_html_docs():
-    html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
-    soup = BeautifulSoup(html_bytes.decode("utf-8", "ignore"), "html.parser")
-
-    return [
-        Document(
-            page_content=soup.get_text("\n"),
-            metadata={"source": "Hochschulgesetz NRW", "url": OFFICIAL_HG_URL},
-        )
-    ]
-
-
-def chunk_docs(docs):
-    splitter = RecursiveCharacterTextSplitter(chunk_size=900, chunk_overlap=80)
-    return splitter.split_documents(docs)
-
-
-# ---------------- Delete old data ----------------
-def delete_old_documents():
-    print("🗑️ Lösche alte Daten…")
-    supabase.table("documents").delete().gte(
-        "id", "00000000-0000-0000-0000-000000000000"
-    ).execute()
-
-
-# ---------------- Ingest ----------------
-def ingest():
-    delete_old_documents()
-
-    pdf_docs = load_pdf_docs()
-    html_docs = load_html_docs()
-
-    chunks = chunk_docs(pdf_docs + html_docs)
-
-    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
-
-    print("📥 Speichere neue Dokumente…")
-    for d in chunks:
-        emb = embeddings.embed_query(d.page_content)
-
-        supabase.table("documents").insert(
-            {
-                "content": d.page_content,
-                "metadata": d.metadata,
-                "embedding": emb,
-            }
-        ).execute()
-
-    print("✅ Ingest abgeschlossen!")
-
-
-if __name__ == "__main__":
-    ingest()

llm.py

@@ -0,0 +1,26 @@
+# llm.py – OpenAI Chatmodell für RAG
+
+import os
+from langchain_openai import ChatOpenAI
+
+CHAT_MODEL = "gpt-4o-mini"  # günstig & stark
+
+def load_llm():
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY fehlt. Bitte als Secret im HuggingFace Space setzen."
+        )
+
+    print(f">>> Lade OpenAI Chatmodell: {CHAT_MODEL}")
+
+    llm = ChatOpenAI(
+        model=CHAT_MODEL,
+        temperature=0.0,   # deterministisch, wenig Halluzination
+        api_key=api_key,
+    )
+    return llm
+
+if __name__ == "__main__":
+    llm = load_llm()
+    print(llm.invoke("Sag einen Satz zum Prüfungsrecht.").content)

load_documents.py

@@ -0,0 +1,119 @@
+
+import os
+import requests
+import tempfile
+from supabase import create_client
+from langchain_core.documents import Document
+from langchain_community.document_loaders import PyPDFLoader
+
+# ---------------------------------------------------------
+# ENV Variablen aus HuggingFace Space
+# ---------------------------------------------------------
+SUPABASE_URL = os.getenv("SUPABASE_URL")
+SUPABASE_ANON_KEY = os.getenv("SUPABASE_ANON_KEY")
+
+if not SUPABASE_URL or not SUPABASE_ANON_KEY:
+    raise RuntimeError("Missing SUPABASE_URL / SUPABASE_ANON_KEY in environment.")
+
+supabase = create_client(SUPABASE_URL, SUPABASE_ANON_KEY)
+
+# ---------------------------------------------------------
+# Prüfungsordnung PDF – liegt in Supabase Storage (public)
+# ---------------------------------------------------------
+PDF_FILE = "f10_bpo_ifb_tei_mif_wii_2021-01-04.pdf"
+PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/File%20PDF/{PDF_FILE}"
+
+# ---------------------------------------------------------
+# Statischer Paragraph-Viewer in HuggingFace Space
+# -> hg_clean.html liegt als Datei im Repo!
+# -> in der App: iframe src="file=hg_clean.html"
+# -> für Links: "file=hg_clean.html#para_123"
+# ---------------------------------------------------------
+# HG_HTML_URL = "file=hg_clean.html"   # WICHTIG: nicht absolut, Space kümmert sich
+#HG_HTML_URL = "https://huggingface.co/spaces/Nguyen5/chatbot/resolve/main/hg_clean.html"
+#HG_HTML_URL = "https://huggingface.co/spaces/Nguyen5/chatbot/raw/main/hg_clean.html"
+
+HG_HTML_URL = f"{SUPABASE_URL}/storage/v1/object/public/hg_viewer/hg_clean.html"
+
+def load_hg_nrw():
+    """
+    Holt alle Paragraphen aus Tabelle public.hg_nrw und baut
+    LangChain-Dokumente. Jeder Paragraph:
+      - title  : z.B. "§ 64 (Fn 35) Prüfungsordnungen"
+      - content: Volltext inkl. Fußnoten
+      - abs_id : para_1, para_2, ...
+    """
+    print(">>> Lade Hochschulgesetz NRW (§) aus Supabase…")
+
+    rows = (
+        supabase.table("hg_nrw")
+        .select("*")
+        .order("order_index")
+        .execute()
+    ).data or []
+
+    print(f"   - {len(rows)} Paragraphen geladen.")
+
+    docs = []
+    for r in rows:
+        abs_id = r["abs_id"]      # z.B. "para_64"
+        title = r.get("title", "")
+        content = r.get("content", "")
+
+        # Anker im Viewer – IDs in hg_clean.html sind identisch (id="para_64")
+        viewer_url = f"{HG_HTML_URL}#{abs_id}"
+
+        docs.append(
+            Document(
+                page_content=f"{title}\n{content}",
+                metadata={
+                    "source": "Hochschulgesetz NRW",
+                    "paragraph": title,
+                    "abs_id": abs_id,
+                    "url": viewer_url,
+                },
+            )
+        )
+
+    return docs
+
+def load_pdf():
+    """
+    Lädt Prüfungsordnung aus Supabase Storage, speichert temporär,
+    splitten erfolgt später in split_documents.py.
+    """
+    print(">>> Lade Prüfungsordnung PDF …")
+
+    resp = requests.get(PDF_URL)
+    resp.raise_for_status()
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
+        tmp.write(resp.content)
+        path = tmp.name
+
+    pages = PyPDFLoader(path).load()
+
+    for i, p in enumerate(pages):
+        p.metadata["source"] = "Prüfungsordnung (PDF)"
+        p.metadata["page"] = i           # 0-basiert
+        p.metadata["pdf_url"] = PDF_URL
+
+    print(f"   - {len(pages)} PDF-Seiten geladen.")
+    return pages
+
+def load_documents():
+    """
+    Master-Funktion:
+      - Hochschulgesetz NRW (Supabase Tabelle hg_nrw)
+      - Prüfungsordnung (PDF)
+    """
+    docs = []
+    docs.extend(load_hg_nrw())
+    docs.extend(load_pdf())
+    print(f"✔ DOCUMENTS LOADED: {len(docs)}")
+    return docs
+
+if __name__ == "__main__":
+    docs = load_documents()
+    print(docs[0])
+    print("Total:", len(docs))

rag_pipeline.py

@@ -1,131 +1,125 @@
-# rag_pipeline.py
-from typing import Any
-from datetime import date
+# rag_pipeline.py – OpenAI RAG mit Supabase-Dokumenten
 
-from supabase_client import supabase, match_documents
+from typing import List, Dict, Any, Tuple
+from langchain_core.messages import SystemMessage, HumanMessage
 
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-from langchain_core.messages import (
-    SystemMessage,
-    HumanMessage,
-    AIMessage,
-)
+MAX_CHARS = 900
 
-emb = OpenAIEmbeddings(model="text-embedding-3-small")
-llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.0)
+# ------------------------------------------------------
+# Quellen-Metadaten
+# ------------------------------------------------------
 
+def build_sources_metadata(docs: List) -> List[Dict[str, Any]]:
+    """
+    Gibt für jeden verwendeten Chunk eine strukturierte Quelle zurück:
+    - id
+    - source (Dokument)
+    - page (bei PDF)
+    - url (PDF-Seite oder hg_clean.html#para_x)
+    - snippet (Kurzvorschau)
+    """
+    srcs = []
 
-SYSTEM_PROMPT = """
-Du bist ein hochpräziser juristischer Assistent für Prüfungsrecht in NRW.
-- Nutze AUSSCHLIESSLICH Dokumente, die über das Tool geliefert werden.
-- Keine Spekulation.
-- Antwort strukturiert + verständlich.
-"""
+    for i, d in enumerate(docs):
+        meta = d.metadata
+        src = meta.get("source")
+        page = meta.get("page")
+        snippet = d.page_content[:300].replace("\n", " ")
+
+        if src == "Prüfungsordnung (PDF)":
+            pdf_url = meta.get("pdf_url")
+            if isinstance(page, int) and pdf_url:
+                url = f"{pdf_url}#page={page + 1}"
+            else:
+                url = pdf_url
+
+        elif src == "Hochschulgesetz NRW":
+            url = meta.get("url")
+            page = None
+
+        else:
+            url = None
+
+        srcs.append({
+            "id": i + 1,
+            "source": src,
+            "page": page + 1 if isinstance(page, int) else None,
+            "url": url,
+            "snippet": snippet,
+        })
 
+    return srcs
 
-# ---------------- TOOL: Suche Dokumente ----------------
-def tool_suche_dokumente(query: str):
-    vec = emb.embed_query(query)
-    docs = match_documents(vec, k=4)
+# ------------------------------------------------------
+# Kontextformatierung
+# ------------------------------------------------------
 
-    results = []
+def format_context(docs):
+    if not docs:
+        return "(Kein relevanter Kontext gefunden.)"
+
+    out_lines = []
     for i, d in enumerate(docs):
-        meta = d["metadata"] or {}
-        snippet = d["content"].replace("\n", " ")[:400]
-
-        results.append(
-            {
-                "index": i + 1,
-                "source": meta.get("source"),
-                "page": meta.get("page"),
-                "snippet": snippet,
-                "content": d["content"],
-                "metadata": meta,
-            }
-        )
-
-    return {"results": results}
-
-
-TOOLS = [
-    {
-        "type": "function",
-        "function": {
-            "name": "suche_pruefungsrecht_dokumente",
-            "description": "Sucht relevante Stellen im Prüfungsrecht.",
-            "parameters": {
-                "type": "object",
-                "properties": {"query": {"type": "string"}},
-                "required": ["query"],
-            },
-        },
-    }
-]
-
-llm_tools = llm.bind_tools(TOOLS)
-
-
-# ---------------- HISTORY LOG ----------------
-def save_message(role: str, content: str):
-    supabase.table("chat_history").insert(
-        {
-            "session_date": date.today().isoformat(),
-            "role": role,
-            "message": content,
-        }
-    ).execute()
-
-
-def convert_history(hist):
-    msgs = []
-    for h in hist[-6:]:
-        if h["role"] == "user":
-            msgs.append(HumanMessage(content=h["content"]))
+        txt = d.page_content[:MAX_CHARS]
+        src = d.metadata.get("source")
+        page = d.metadata.get("page")
+
+        if src == "Prüfungsordnung (PDF)" and isinstance(page, int):
+            src_str = f"{src}, Seite {page + 1}"
         else:
-            msgs.append(AIMessage(content=h["content"]))
-    return msgs
+            src_str = src
 
+        out_lines.append(f"[KONTEXT {i+1}] ({src_str})\n{txt}")
 
-# ---------------- AGENT ANSWER ----------------
-def agent_answer(query: str, history: Any):
-    messages = [
-        SystemMessage(content=SYSTEM_PROMPT),
-        *convert_history(history),
-        HumanMessage(content=query),
-    ]
+    return "\n\n".join(out_lines)
 
-    first = llm_tools.invoke(messages)
-
-    if first.tool_calls:
-        call = first.tool_calls[0]
-        if call["name"] == "suche_pruefungsrecht_dokumente":
-            tool_res = tool_suche_dokumente(call["args"]["query"])
-
-            messages.extend(
-                [
-                    first,
-                    AIMessage(
-                        content=str(tool_res),
-                        name="suche_pruefungsrecht_dokumente",
-                    ),
-                ]
-            )
-
-            final = llm.invoke(messages)
-            answer = final.content
-            docs = tool_res["results"]
-        else:
-            answer = "Tool nicht unterstützt."
-            docs = []
-    else:
-        answer = first.content
-        docs = []
+SYSTEM_PROMPT = """
+Du bist ein juristisch präziser Chatbot für Prüfungsrecht.
+Du nutzt ausschließlich:
 
-    save_message("user", query)
-    save_message("assistant", answer)
+- die Prüfungsordnung (PDF) und
+- das Hochschulgesetz NRW (Paragraphen aus der Datenbank / hg_clean.html)
+
+Regeln:
+
+1. Antworte nur auf Basis des gelieferten Kontextes.
+2. Wenn der Kontext keine sichere Antwort erlaubt, sage das klar.
+3. Antworte in gut verständlichem Deutsch, in ganzen Sätzen.
+4. Nenne, soweit möglich:
+   - Paragraphen oder Überschriften,
+   - das Dokument (Prüfungsordnung / Hochschulgesetz NRW),
+   - Seitenzahl (bei der Prüfungsordnung).
+"""
+
+def answer(question: str, retriever, chat_model) -> Tuple[str, List[Dict[str, Any]]]:
+    # 1. Chunks holen
+    docs = retriever.invoke(question)
+    context_str = format_context(docs)
+
+    # 2. Prompt bauen
+    human = f"""
+FRAGE:
+{question}
+
+NUTZE AUSSCHLIESSLICH DIESEN KONTEXT:
+{context_str}
+
+AUFGABE:
+Erstelle eine juristisch korrekte Antwort ausschließlich auf Basis
+des obigen Kontextes. Wenn der Kontext keine sichere Antwort zulässt,
+sage das ausdrücklich und verzichte auf Spekulationen.
+"""
+
+    msgs = [
+        SystemMessage(content=SYSTEM_PROMPT),
+        HumanMessage(content=human),
+    ]
 
-    return answer, docs
+    # 3. LLM aufrufen
+    result = chat_model.invoke(msgs)
+    answer_text = result.content.strip()
 
+    # 4. Quellenliste
+    sources = build_sources_metadata(docs)
 
-def rag_answer(query: str, history: Any):
-    return agent_answer(query, history)
+    return answer_text, sources

requirements.txt

@@ -1,10 +1,32 @@
+# === UI ===
+gradio
+
+# === Supabase ===
+supabase
+postgrest
+httpx
+python-dotenv
+
+# === LangChain Core ===
 langchain
 langchain-community
+langchain-text-splitters
 langchain-openai
-openai
-supabase
-gradio
+
+# === VectorStore ===
+faiss-cpu
+
+# === PDF + HTTP + HTML ===
 pypdf
+requests
 beautifulsoup4
-python-dotenv
-soundfile
+
+# === Audio (STT/TTS local) ===
+transformers
+accelerate
+soundfile
+scipy
+numpy
+
+# OpenAI offizielle Bibliothek (kommt i.d.R. mit langchain-openai, zur Sicherheit explizit)
+openai

retriever.py

@@ -0,0 +1,47 @@
+"""
+BƯỚC 5: RETRIEVER
+-----------------
+Tạo LangChain Retriever từ FAISS VectorStore.
+
+Retriever sẽ dùng trong bước RAG sau này:
+- retriever.get_relevant_documents(query)
+"""
+
+from langchain_community.vectorstores import FAISS
+
+# số chunk sẽ lấy cho mỗi câu hỏi
+RETRIEVER_K = 4
+
+def get_retriever(vectorstore: FAISS, k: int = RETRIEVER_K):
+    """
+    Tạo retriever từ FAISS VectorStore.
+    """
+    print(f">>> Creating retriever with k={k} ...")
+    retriever = vectorstore.as_retriever(search_kwargs={"k": k})
+    print(">>> Retriever ready.\n")
+    return retriever
+
+if __name__ == "__main__":
+    # Test: load -> split -> FAISS -> retriever.get_relevant_documents()
+    from load_documents import load_documents
+    from split_documents import split_documents
+    from vectorstore import build_vectorstore
+
+    print("=== TEST: retriever.get_relevant_documents ===\n")
+
+    docs = load_documents()
+    chunks = split_documents(docs)
+    vs = build_vectorstore(chunks)
+    retriever = get_retriever(vs, k=4)
+
+    query = "Wie lange habe ich Zeit, eine Prüfungsleistung zu wiederholen?"
+    print("Test query:")
+    print(" ", query, "\n")
+
+    retrieved_docs = retriever.invoke(query)
+
+    print(f"Retriever returned {len(retrieved_docs)} documents.")
+    for i, d in enumerate(retrieved_docs, start=1):
+        print(f"\n=== DOC {i} ===")
+        print(d.page_content[:400], "...")
+        print("Metadata:", d.metadata)

speech_io.py

@@ -0,0 +1,157 @@
+"""
+speech_io.py
+
+Sprachbasierte Ein-/Ausgabe:
+- Speech-to-Text (STT) mit Whisper (transformers.pipeline)
+- Text-to-Speech (TTS) mit MMS-TTS Deutsch
+
+Dieses File ist 100% stabil für HuggingFace Spaces.
+"""
+
+from typing import Optional, Tuple
+import numpy as np
+import soundfile as sf
+from scipy.signal import butter, filtfilt
+from transformers import pipeline
+
+# Modelle
+ASR_MODEL_ID = "openai/whisper-small"        
+TTS_MODEL_ID = "facebook/mms-tts-deu"       
+
+_asr = None
+_tts = None
+
+# ========================================================
+# STT PIPELINE
+# ========================================================
+
+def get_asr_pipeline():
+    global _asr
+    if _asr is None:
+        print(f">>> Lade ASR Modell: {ASR_MODEL_ID}")
+        _asr = pipeline(
+            task="automatic-speech-recognition",
+            model=ASR_MODEL_ID,
+            device="cpu",
+            return_timestamps=True,   # wichtig
+            chunk_length_s=30         # auto-chunk für lange audio
+        )
+    return _asr
+
+# ========================================================
+# TTS PIPELINE
+# ========================================================
+
+def get_tts_pipeline():
+    global _tts
+    if _tts is None:
+        print(f">>> Lade TTS Modell: {TTS_MODEL_ID}")
+        _tts = pipeline(
+            task="text-to-speech",
+            model=TTS_MODEL_ID,
+        )
+    return _tts
+
+# ========================================================
+# AUDIO FILTER – Noise Reduction + Highpass
+# ========================================================
+
+def butter_highpass_filter(data, cutoff=60, fs=16000, order=4):
+    nyq = 0.5 * fs
+    norm_cutoff = cutoff / nyq
+    b, a = butter(order, norm_cutoff, btype="high")
+    return filtfilt(b, a, data)
+
+def apply_fade(audio, sr, duration_ms=10):
+    fade_samples = int(sr * duration_ms / 1000)
+
+    if fade_samples * 2 >= len(audio):
+        return audio
+
+    fade_in_curve = np.linspace(0, 1, fade_samples)
+    audio[:fade_samples] *= fade_in_curve
+
+    fade_out_curve = np.linspace(1, 0, fade_samples)
+    audio[-fade_samples:] *= fade_out_curve
+
+    return audio
+
+# ========================================================
+# SPEECH-TO-TEXT (STT)
+# ========================================================
+
+def transcribe_audio(audio_path: str) -> str:
+    """
+    audio_path: path zu WAV-Datei (von gr.Audio type="filepath")
+    """
+
+    if audio_path is None:
+        return ""
+
+    # WAV einlesen (soundfile garantiert PCM korrekt)
+    data, sr = sf.read(audio_path)
+
+    # immer Mono
+    if len(data.shape) > 1:
+        data = data[:, 0]
+
+    # Whisper >30s vermeiden
+    MAX_SAMPLES = sr * 30
+    if len(data) > MAX_SAMPLES:
+        data = data[:MAX_SAMPLES]
+
+    asr = get_asr_pipeline()
+
+    print(">>> Transkribiere Audio...")
+    result = asr(
+        {"array": data, "sampling_rate": sr},
+    )
+
+    text = result.get("text", "").strip()
+    print("ASR:", text)
+    return text
+
+# ========================================================
+# TEXT-TO-SPEECH (TTS)
+# ========================================================
+
+def synthesize_speech(text: str):
+    if not text or not text.strip():
+        return None
+
+    tts = get_tts_pipeline()
+    out = tts(text)
+
+    # rohes Audio from MMS (float32 [-1, 1])
+    audio = np.array(out["audio"], dtype=np.float32)
+    sr = out.get("sampling_rate", 16000)
+
+    # ===== FIX sample_rate =====
+    if sr is None or sr <= 0 or sr > 65535:
+        sr = 16000
+
+    # ===== Mono erzwingen =====
+    if audio.ndim > 1:
+        audio = audio.squeeze()
+    if audio.ndim > 1:
+        audio = audio[:, 0]
+
+    # ===== Noise reduction =====
+    try:
+        audio = butter_highpass_filter(audio, cutoff=60, fs=sr)
+    except:
+        pass
+
+    # ===== Normalize =====
+    max_val = np.max(np.abs(audio))
+    if max_val > 0:
+        audio = audio / max_val
+
+    # ===== Fade gegen pop =====
+    audio = apply_fade(audio, sr)
+
+    # ===== int16 =====
+    audio_int16 = np.clip(audio * 32767, -32768, 32767).astype(np.int16)
+
+    # Rückgabe: (sr, np.int16 array)
+    return (sr, audio_int16)

split_documents.py

@@ -0,0 +1,28 @@
+# split_documents.py – v2
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+CHUNK_SIZE = 1500
+CHUNK_OVERLAP = 200
+
+def split_documents(docs):
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
+    chunks = splitter.split_documents(docs)
+
+    for c in chunks:
+        c.metadata["chunk_size"] = CHUNK_SIZE
+        c.metadata["chunk_overlap"] = CHUNK_OVERLAP
+
+    return chunks
+
+if __name__ == "__main__":
+    from load_documents import load_documents
+    docs = load_documents()
+    chunks = split_documents(docs)
+    print("Docs:", len(docs), "Chunks:", len(chunks))
+    print(chunks[0].page_content[:300], chunks[0].metadata)
+

supabase_client.py

@@ -1,25 +0,0 @@
-# supabase_client.py
-import os
-from supabase import create_client
-
-SUPABASE_URL = os.environ["SUPABASE_URL"]
-SUPABASE_SERVICE_ROLE = os.environ["SUPABASE_SERVICE_ROLE"]
-
-supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
-
-
-def load_file_bytes(bucket: str, filename: str) -> bytes:
-    return supabase.storage.from_(bucket).download(filename)
-
-
-def match_documents(embedding: list, k: int = 4):
-    """
-    Gọi trực tiếp RPC match_documents trong Supabase.
-    Trả về list các rows: {content, metadata, embedding?}
-    """
-    resp = supabase.rpc(
-        "match_documents",
-        {"query_embedding": embedding, "match_count": k}
-    ).execute()
-
-    return resp.data or []

upload_weblink_to_supabase.py

@@ -0,0 +1,76 @@
+import os
+import requests
+from bs4 import BeautifulSoup
+from supabase import create_client
+from dotenv import load_dotenv
+
+load_dotenv()
+
+SUPABASE_URL = os.environ["SUPABASE_URL"]
+SUPABASE_SERVICE_ROLE = os.environ["SUPABASE_SERVICE_ROLE"]
+
+supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
+
+LAW_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
+
+def extract_paragraphs():
+    print(">>> Lade Hochschulgesetz NRW …")
+
+    html = requests.get(LAW_URL, timeout=30).text
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Tất cả tiêu đề Paragraph xuất hiện trong <h2> hoặc <h3>
+    headers = soup.find_all(["h2", "h3"])
+
+    paragraphs = []
+    order = 1
+
+    for header in headers:
+        title = header.get_text(" ", strip=True)
+
+        if not title.startswith("§"):
+            continue  # bỏ các h2/h3 không phải Paragraph
+
+        # Gom toàn bộ nội dung từ header đến trước h2/h3 tiếp theo
+        content_parts = []
+        sibling = header.find_next_sibling()
+
+        while sibling and sibling.name not in ["h2", "h3"]:
+            text = sibling.get_text(" ", strip=True)
+            if text:
+                content_parts.append(text)
+            sibling = sibling.find_next_sibling()
+
+        full_content = "\n".join(content_parts).strip()
+
+        para_id = f"para_{order}"
+
+        paragraphs.append({
+            "abs_id": para_id,
+            "title": title,
+            "content": full_content,
+            "order_index": order
+        })
+
+        order += 1
+
+    print(f"✔ Extracted {len(paragraphs)} paragraphs (§).")
+    return paragraphs
+
+def upload_to_supabase():
+    paras = extract_paragraphs()
+
+    print(">>> Clear table hg_nrw …")
+    supabase.table("hg_nrw").delete().neq("abs_id", "").execute()
+
+    print(">>> Upload begin …")
+    BATCH = 100
+    for i in range(0, len(paras), BATCH):
+        batch = paras[i:i+BATCH]
+        print(f"   - Upload batch {i} – {i+len(batch)-1}")
+        supabase.table("hg_nrw").upsert(batch).execute()
+
+    print("✔ DONE uploading complete NRW law.")
+
+if __name__ == "__main__":
+    upload_to_supabase()

vectorstore.py

@@ -0,0 +1,56 @@
+"""
+BƯỚC 4: VECTORSTORE (FAISS in-memory)
+-------------------------------------
+Tạo FAISS index từ các CHUNK văn bản.
+
+- Không ghi file .faiss nào, tất cả nằm trong RAM.
+- Embeddings được lấy từ get_embeddings() (Bước 3).
+"""
+
+from langchain_community.vectorstores import FAISS
+from embeddings import get_embeddings
+
+def build_vectorstore(chunks):
+    """
+    Nhận danh sách Document (đã split) và trả về FAISS VectorStore.
+    """
+    print(">>> Initialising embedding model for FAISS index ...")
+    embeddings = get_embeddings()
+
+    print(f">>> Building FAISS index from {len(chunks)} chunks ...")
+    vs = FAISS.from_documents(chunks, embeddings)
+    print(">>> FAISS index built.\n")
+    return vs
+
+if __name__ == "__main__":
+    # Test toàn pipeline: load -> split -> FAISS -> similarity_search
+    from load_documents import load_documents
+    from split_documents import split_documents
+
+    print("=== TEST: load_documents -> split_documents -> FAISS.similarity_search ===\n")
+
+    # 1) Load tài liệu (PDF + HTML) từ HuggingFace
+    docs = load_documents()
+
+    # 2) Split thành chunks
+    from pprint import pprint
+    print(f"Loaded {len(docs)} raw documents.")
+    chunks = split_documents(docs)
+    print(f"Split into {len(chunks)} chunks.\n")
+
+    # 3) Xây FAISS vectorstore
+    vectorstore = build_vectorstore(chunks)
+
+    # 4) Test similarity_search
+    query = "Fristen für die Prüfungsanmeldung im Bachelorstudium"
+    print("Test query:")
+    print(" ", query, "\n")
+
+    results = vectorstore.similarity_search(query, k=3)
+
+    print("Top-3 ähnliche Chunks aus dem VectorStore:")
+    for i, doc in enumerate(results, start=1):
+        print(f"\n=== RESULT {i} ===")
+        print(doc.page_content[:400], "...")
+        print("Metadata:", doc.metadata)
+