commit

app.py

@@ -0,0 +1,205 @@
+# app.py – Prüfungsrechts-Chatbot (RAG + Sprache, UI kiểu ChatGPT)
+
+import gradio as gr
+from gradio_pdf import PDF
+
+from load_documents import load_all_documents
+from split_documents import split_documents
+from vectorstore import build_vectorstore
+from retriever import get_retriever
+from llm import load_llm
+from rag_pipeline import answer
+from speech_io import transcribe_audio, synthesize_speech
+
+# =====================================================
+# INITIALISIERUNG (global)
+# =====================================================
+
+print("📚 Lade Dokumente…")
+docs = load_all_documents()
+
+print("🔪 Splitte Dokumente…")
+chunks = split_documents(docs)
+
+print("🔍 Erstelle VectorStore…")
+vs = build_vectorstore(chunks)
+
+print("🔎 Erzeuge Retriever…")
+retriever = get_retriever(vs)
+
+print("🤖 Lade LLM…")
+llm = load_llm()
+
+
+# =====================================================
+# Quellen formatieren – Markdown für Chat
+# =====================================================
+def format_sources(src):
+    if not src:
+        return ""
+
+    out = ["", "## 📚 Quellen"]
+
+    for s in src:
+        line = f"- [{s['source']}]({s['url']})"
+        if s.get("page") is not None:
+            line += f" (Seite {s['page']})"
+        out.append(line)
+
+    return "\n".join(out)
+
+
+# =====================================================
+# CORE CHAT-FUNKTION (MultimodalTextbox: Text + Audio)
+# =====================================================
+def chat_fn(message, history):
+    """
+    message: dict {"text": str, "files": [...]} von gr.MultimodalTextbox
+    history: Liste von OpenAI-ähnlichen Messages (role, content)
+    """
+    # 1) Text + evtl. Audio aus message holen
+    if isinstance(message, dict):
+        text = (message.get("text") or "").strip()
+        files = message.get("files") or []
+    else:
+        text = str(message or "").strip()
+        files = []
+
+    # Audio-Datei (vom Mikrofon) herausziehen
+    audio_path = None
+    for f in files:
+        # gr.MultimodalTextbox liefert i.d.R. Dict mit "path"
+        if isinstance(f, dict):
+            path = f.get("path")
+        else:
+            path = f
+        if isinstance(path, str) and path:
+            audio_path = path
+            break
+
+    # Wenn Audio vorhanden: transkribieren
+    if audio_path:
+        spoken = transcribe_audio(audio_path)
+        if text:
+            text = (text + " " + spoken).strip()
+        else:
+            text = spoken
+
+    if not text:
+        # Nichts zu tun
+        return history, None, {"text": "", "files": []}
+
+    # 2) RAG-Antwort berechnen
+    ans, sources = answer(text, retriever, llm)
+    bot_msg = ans + format_sources(sources)
+
+    # 3) History aktualisieren (ChatGPT-Style)
+    history = history + [
+        {"role": "user", "content": text},
+        {"role": "assistant", "content": bot_msg},
+    ]
+
+    # 4) TTS für Antwort
+    tts_audio = synthesize_speech(bot_msg)
+
+    # 5) Input-Feld leeren
+    cleared_input = {"text": "", "files": []}
+
+    return history, tts_audio, cleared_input
+
+
+# =====================================================
+# LAST ANSWER → TTS (für Button "Antwort erneut vorlesen")
+# =====================================================
+def read_last_answer(history):
+    if not history:
+        return None
+
+    for msg in reversed(history):
+        if msg.get("role") == "assistant":
+            return synthesize_speech(msg.get("content", ""))
+
+    return None
+
+
+# =====================================================
+# UI – GRADIO
+# =====================================================
+with gr.Blocks(title="Prüfungsrechts-Chatbot (RAG + Sprache)") as demo:
+    gr.Markdown("# 🧑‍⚖️ Prüfungsrechts-Chatbot")
+    gr.Markdown(
+        "Dieser Chatbot beantwortet Fragen **ausschließlich** aus der "
+        "Prüfungsordnung (PDF) und dem Hochschulgesetz NRW. "
+        "Du kannst Text eingeben oder direkt ins Mikrofon sprechen."
+    )
+
+    with gr.Row():
+        # ===================== LINKER TEIL: Chat =====================
+        with gr.Column(scale=2):
+            chatbot = gr.Chatbot(
+                label="Chat",
+                height=500,
+            )
+
+            # Audio-Ausgabe (TTS)
+            voice_out = gr.Audio(label="Vorgelesene Antwort", type="numpy")
+
+            # Multimodal-Textbox mit Mikrofon in der Leiste
+            chat_input = gr.MultimodalTextbox(
+                label=None,
+                placeholder="Stelle deine Frage zum Prüfungsrecht … oder sprich ins Mikrofon",
+                show_label=False,
+                sources=["microphone"],  # nur Mikrofon (kein Upload nötig)
+                file_types=["audio"],
+                max_lines=6,
+            )
+
+            # Senden bei Enter / Klick auf Icon
+            chat_input.submit(
+                chat_fn,
+                [chat_input, chatbot],
+                [chatbot, voice_out, chat_input],
+            )
+
+            send_btn = gr.Button("Senden")
+            send_btn.click(
+                chat_fn,
+                [chat_input, chatbot],
+                [chatbot, voice_out, chat_input],
+            )
+
+            # Button: Antwort erneut vorlesen
+            read_btn = gr.Button("🔁 Antwort erneut vorlesen")
+            read_btn.click(
+                read_last_answer,
+                [chatbot],
+                [voice_out],
+            )
+
+            # Chat löschen
+            clear_btn = gr.Button("Chat zurücksetzen")
+            clear_btn.click(
+                lambda: ([], None, {"text": "", "files": []}),
+                None,
+                [chatbot, voice_out, chat_input],
+            )
+
+        # ===================== RECHTER TEIL: Viewer =====================
+        with gr.Column(scale=1):
+            # PDF-URL aus metadata holen
+            pdf_meta = next(d.metadata for d in docs if d.metadata["type"] == "pdf")
+            gr.Markdown("### 📄 Prüfungsordnung (PDF)")
+            PDF(pdf_meta["pdf_url"], height=350)
+
+            # HG-Viewer-URL (hg_clean.html aus Supabase Storage)
+            hg_meta = next(d.metadata for d in docs if d.metadata["type"] == "hg")
+            hg_url = hg_meta["viewer_url"].split("#")[0]
+
+            gr.Markdown("### 📘 Hochschulgesetz NRW (Viewer)")
+            gr.HTML(
+                f'<iframe src="{hg_url}" '
+                'style="width:100%;height:350px;border:none;"></iframe>'
+            )
+
+if __name__ == "__main__":
+    demo.queue().launch(ssr_mode=False, show_error=True)

build_hg_viewer.py

@@ -0,0 +1,285 @@
+# build_hg_viewer.py
+import os
+from supabase import create_client
+from dotenv import load_dotenv
+
+load_dotenv()
+
+SUPABASE_URL = os.environ["SUPABASE_URL"]
+SUPABASE_SERVICE_ROLE = os.environ["SUPABASE_SERVICE_ROLE"]
+
+if not SUPABASE_URL or not SUPABASE_SERVICE_ROLE:
+    raise RuntimeError("Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE")
+
+supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
+
+from upload_weblink_to_supabase import extract_paragraphs
+
+# ======== HTML TEMPLATE ========
+VIEW_TEMPLATE = """
+<!DOCTYPE html>
+<html lang="de">
+<head>
+<meta charset="UTF-8">
+<title>Hochschulgesetz NRW – Paragraph Viewer</title>
+<style>
+body {
+    font-family: Arial, sans-serif;
+    margin: 0;
+    padding: 0;
+    display: flex;
+}
+/* ----------- SIDEBAR ------------- */
+#sidebar {
+    width: 280px;
+    height: 100vh;
+    overflow-y: auto;
+    background: #f5f5f5;
+    border-right: 1px solid #ccc;
+    padding: 15px;
+    position: sticky;
+    top: 0;
+}
+#sidebar h2 {
+    margin-top: 0;
+}
+#searchBox {
+    width: 100%;
+    padding: 8px;
+    font-size: 15px;
+    margin-bottom: 10px;
+    border: 1px solid #aaa;
+    border-radius: 5px;
+}
+.sidebar-link {
+    display: block;
+    padding: 6px 8px;
+    margin-bottom: 4px;
+    text-decoration: none;
+    color: #003366;
+    border-radius: 4px;
+}
+.sidebar-link:hover {
+    background: #e0e7ff;
+    color: #001d4d;
+}
+/* ----------- CONTENT ------------- */
+#content {
+    flex: 1;
+    padding: 25px;
+    max-width: 900px;
+}
+/* Absatz block */
+.para {
+    padding: 20px 0;
+    border-bottom: 1px solid #ddd;
+}
+.para h2 {
+    color: #003366;
+    margin-bottom: 10px;
+}
+/* ----------- Fußnoten ------------- */
+.fn-block {
+    background: #fafafa;
+    border-left: 4px solid #999;
+    padding: 12px;
+    margin-top: 10px;
+    margin-bottom: 25px;
+}
+.fn-toggle {
+    cursor: pointer;
+    font-weight: bold;
+    color: #003366;
+    margin-bottom: 5px;
+}
+.fn-content {
+    display: none;
+    padding-left: 10px;
+}
+.fn-title {
+    font-weight: bold;
+    margin-bottom: 6px;
+}
+.fn-item {
+    margin-bottom: 8px;
+}
+/* ----------- Highlight beim Öffnen ------------- */
+.highlight {
+    animation: flash 2s ease-in-out;
+    background: #fff8c6 !important;
+}
+@keyframes flash {
+    0% { background: #fff8c6; }
+    100% { background: transparent; }
+}
+/* Keyword highlight */
+.keyword {
+    background: yellow;
+    padding: 2px 3px;
+    border-radius: 3px;
+}
+/* Back to top button */
+#topBtn {
+    position: fixed;
+    bottom: 25px;
+    right: 25px;
+    background: #003366;
+    color: white;
+    border-radius: 8px;
+    padding: 10px 14px;
+    cursor: pointer;
+    font-size: 16px;
+    display: none;
+}
+</style>
+</head>
+<body>
+<div id="sidebar">
+    <h2>Inhaltsverzeichnis</h2>
+    <input type="text" id="searchBox" placeholder="Suchen nach § …">
+    <!-- SIDEBAR_LINKS -->
+</div>
+<div id="content">
+    <h1>Hochschulgesetz NRW – Paragraph Viewer</h1>
+    <!-- PARAGRAPH_CONTENT -->
+</div>
+<div id="topBtn" onclick="scrollToTop()">⬆️ Top</div>
+<script>
+// ------ TỰ ĐỘNG HIGHLIGHT Absatz khi có #anchor HIGHLIGHT ABSATZ & SCROLL ------
+window.onload = function() {
+    const anchor = window.location.hash.substring(1);
+    const params = new URLSearchParams(window.location.search);
+    const keywords = params.get("k");
+    if (anchor) {
+        const el = document.getElementById(anchor);
+        if (el) {
+            el.classList.add("highlight");
+            el.scrollIntoView({ behavior: "smooth", block: "center" });
+        }
+    }
+    /* KEYWORD HIGHLIGHT */
+    if (keywords) {
+        const words = keywords.split("%20");
+        highlightKeywords(words);
+    }
+};
+/* --- KEYWORD HIGHLIGHT FUNCTION --- */
+function highlightKeywords(words) {
+    const container = document.getElementById("content");
+    let html = container.innerHTML;
+    words.forEach(word => {
+        if (word.length < 2) return;
+        const regex = new RegExp(`(${decodeURIComponent(word)})`, "gi");
+        html = html.replace(regex, `<span class="keyword">$1</span>`);
+    });
+    container.innerHTML = html;
+}
+/* --- SEARCH IN SIDEBAR --- */
+document.getElementById("searchBox").addEventListener("input", function() {
+    const q = this.value.toLowerCase();
+    document.querySelectorAll(".sidebar-link").forEach(link => {
+        const txt = link.innerText.toLowerCase();
+        link.style.display = txt.includes(q) ? "block" : "none";
+    });
+});
+/* --- COLLAPSIBLE FUSSNOTEN --- */
+document.addEventListener("click", function(e) {
+    if (e.target.classList.contains("fn-toggle")) {
+        const content = e.target.nextElementSibling;
+        content.style.display = content.style.display === "block" ? "none" : "block";
+    }
+});
+/* --- BACK TO TOP BUTTON --- */
+window.onscroll = function() {
+    document.getElementById("topBtn").style.display =
+        window.scrollY > 300 ? "block" : "none";
+};
+function scrollToTop() {
+    window.scrollTo({ top: 0, behavior: 'smooth' });
+}
+</script>
+</body>
+</html>
+"""
+
+# -------------------------------------------------------------------
+# 2. BUILD VIEWER
+# -------------------------------------------------------------------
+
+def build_html_from_db():
+    """
+    Liest alle Paragraphen aus hg_nrw und baut daraus HTML.
+    """
+    print(">>> Lade Paragraphen aus Supabase (hg_nrw) …")
+    paras = extract_paragraphs()
+    # 5.12_2:13
+    res = supabase.table("hg_nrw").select("*").order("order_index").execute()
+    rows = res.data or []
+
+    sidebar_links = ""
+    content_html = ""
+
+    for p in paras:
+        pid = p["abs_id"]
+        title = p["title"]
+        body = p["content"]
+
+        # Sidebar item
+        sidebar_links += f'<a class="sidebar-link" href="#{pid}">{title}</a>\n'
+
+        # Fußnoten tách riêng (bắt đầu bằng "Fn 1", "Fn 2", ...)
+        lines = body.split("\n")
+        main_text = []
+        fn_text = []
+        in_fn = False
+
+        for line in lines:
+            if line.startswith("Fn "):
+                in_fn = True
+            if in_fn:
+                fn_text.append(line)
+            else:
+                main_text.append(line)
+
+        footnotes_html = ""
+        if fn_text:
+            footnotes_html += '<div class="fn-block">'
+            footnotes_html += '<div class="fn-title">Fußnoten:</div>'
+            for fn in fn_text:
+                footnotes_html += f'<div class="fn-item">{fn}</div>'
+            footnotes_html += "</div>"
+
+        # Paragraph block
+        content_html += f"""
+<div class="para" id="{pid}">
+    <h2>{title}</h2>
+    <div>{'<br>'.join(main_text)}</div>
+    {footnotes_html}
+</div>
+"""
+
+    html = VIEW_TEMPLATE.replace("<!-- SIDEBAR_LINKS -->", sidebar_links)
+    html = html.replace("<!-- PARAGRAPH_CONTENT -->", content_html)
+
+    return html
+
+# -------------------------------------------------------------------
+# 3. UPLOAD TO SUPABASE STORAGE
+# -------------------------------------------------------------------
+
+def upload_html():
+    html = build_html_from_db()
+
+    supabase.storage.from_("hg_viewer").update(
+        "hg_clean.html",
+        html.encode("utf-8"),
+        {
+            "content-type": "text/html",
+            "x-upsert": "true"
+        }
+    )
+
+    print("✔ hg_clean.html uploaded!")
+
+if __name__ == "__main__":
+    upload_html()

embeddings.py

@@ -0,0 +1,24 @@
+# embeddings.py – OpenAI Version (text-embedding-3-small)
+
+import os
+from langchain_openai import OpenAIEmbeddings
+
+EMBED_MODEL = "text-embedding-3-small"
+
+def get_embeddings():
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY fehlt. Bitte als Secret im HuggingFace Space setzen."
+        )
+
+    print(f">>> Lade OpenAI Embedding Model: {EMBED_MODEL}")
+    emb = OpenAIEmbeddings(
+        model=EMBED_MODEL,
+        api_key=api_key,
+    )
+    return emb
+
+if __name__ == "__main__":
+    e = get_embeddings()
+    print(e.embed_query("Test"))

llm.py

@@ -0,0 +1,36 @@
+# llm.py – OpenAI Chatmodell für RAG
+
+import os
+from langchain_openai import ChatOpenAI
+
+CHAT_MODEL = "gpt-4o-mini"  # günstig & stark
+
+def load_llm():
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY fehlt. Bitte als Secret im HuggingFace Space setzen."
+        )
+
+    print(f">>> Lade OpenAI Chatmodell: {CHAT_MODEL}")
+
+    # llm = ChatOpenAI(
+    #     model=CHAT_MODEL,
+    #     temperature=0.0,   # deterministisch, wenig Halluzination
+    #     api_key=api_key,
+    # )
+    # return llm
+    # 5.12_2:13
+    llm = ChatOpenAI(
+        model=CHAT_MODEL,
+        temperature=0.0,
+        top_p=1.0,
+        presence_penalty=0.0,
+        frequency_penalty=0.0,
+        api_key=api_key,
+    )
+    return llm
+
+if __name__ == "__main__":
+    llm = load_llm()
+    print(llm.invoke("Sag einen Satz zum Prüfungsrecht.").content)

load_documents.py

@@ -0,0 +1,128 @@
+"""
+LOAD_DOCUMENTS – SINGLE SOURCE OF TRUTH
+Nhiệm vụ:
+1) Lade Prüfungsordnung PDF direkt aus Supabase-Storage.
+2) Lade Hochschulgesetz NRW aus Supabase-Tabelle hg_nrw.
+3) Cung cấp metadata đầy đủ để các file khác KHÔNG PHẢI tính lại URL.
+"""
+
+import os
+import tempfile
+from dotenv import load_dotenv
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_core.documents import Document
+from supabase import create_client
+
+load_dotenv()
+
+import urllib.parse
+
+# ===== Supabase config =====
+SUPABASE_URL = os.getenv("SUPABASE_URL")
+SUPABASE_SERVICE_ROLE = os.getenv("SUPABASE_SERVICE_ROLE")
+
+supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
+
+# ===== Storage Config =====
+
+#import urllib.parse
+PDF_FILE = "f10_bpo_ifb_tei_mif_wii_2021-01-04.pdf"
+
+PDF_BUCKET = "File PDF"
+ENC_BUCKET = urllib.parse.quote(PDF_BUCKET)   # "File%20PDF"
+
+#PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{PDF_BUCKET}/{PDF_FILE}"
+PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{ENC_BUCKET}/{PDF_FILE}"
+
+
+# ===== Viewer URL =====
+HG_VIEWER_BUCKET = "hg_viewer"
+HG_VIEWER_FILE = "hg_clean.html"
+HG_VIEWER_URL = f"{SUPABASE_URL}/storage/v1/object/public/{HG_VIEWER_BUCKET}/{HG_VIEWER_FILE}"
+
+
+# ============================================================
+# 1) PDF aus Supabase laden
+# ============================================================
+
+def load_pdf_from_supabase() -> list[Document]:
+    print("📥 Lade Prüfungsordnung PDF aus Supabase...")
+
+    response = supabase.storage.from_(PDF_BUCKET).download(PDF_FILE)
+    if response is None:
+        raise ValueError("❌ Konnte PDF nicht laden!")
+
+    # Temporäre Datei
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
+        tmp.write(response)
+        temp_pdf_path = tmp.name
+
+    pages = PyPDFLoader(temp_pdf_path).load()
+
+    for i, p in enumerate(pages):
+        p.metadata = {
+            "type": "pdf",
+            "source": "Prüfungsordnung",
+            "page": i,
+            "pdf_url": f"{PDF_URL}#page={i}",
+            "filename": PDF_FILE,
+        }
+
+    print(f"✔ {len(pages)} PDF-Seiten geladen.")
+    return pages
+
+
+# ============================================================
+# 2) HG aus Tabelle laden
+# ============================================================
+
+def load_hg_from_supabase() -> list[Document]:
+    print("📥 Lade Hochschulgesetz NRW aus Tabelle hg_nrw...")
+
+    res = (
+        supabase.table("hg_nrw")
+        .select("*")
+        .order("order_index", desc=False)
+        .execute()
+    )
+    rows = res.data or []
+    docs = []
+
+    for row in rows:
+        abs_id = row["abs_id"]
+        title = row["title"]
+        content = row["content"]
+
+        viewer_url = f"{HG_VIEWER_URL}#{abs_id}"
+
+        docs.append(
+            Document(
+                page_content=content,
+                metadata={
+                    "type": "hg",
+                    "source": "Hochschulgesetz NRW",
+                    "abs_id": abs_id,
+                    "title": title,
+                    "viewer_url": viewer_url,
+                },
+            )
+        )
+
+    print(f"✔ {len(docs)} HG-Absätze geladen.")
+    return docs
+
+
+# ============================================================
+# 3) ALLES LADEN
+# ============================================================
+
+def load_all_documents():
+    pdf_docs = load_pdf_from_supabase()
+    hg_docs = load_hg_from_supabase()
+    return pdf_docs + hg_docs
+
+
+if __name__ == "__main__":
+    docs = load_all_documents()
+    print("📚 Gesamt:", len(docs))
+    print("🔎 Beispiel metadata:", docs[0].metadata)

rag_pipeline.py

@@ -0,0 +1,153 @@
+"""
+RAG PIPELINE – Version 26.11 (ohne Modi, stabil, juristisch korrekt)
+"""
+
+# from typing import List, Dict, Any, Tuple
+# from langchain_core.messages import SystemMessage, HumanMessage
+# from load_documents import DATASET, PDF_FILE, HTML_FILE
+
+# from typing import List, Dict, Any, Tuple
+# import os
+# from langchain_core.messages import SystemMessage, HumanMessage
+# from load_documents import DATASET, PDF_FILE
+
+# 5.12_2:13
+from typing import List, Dict, Any, Tuple
+from langchain_core.messages import SystemMessage, HumanMessage
+
+MAX_CHARS = 900
+
+# ============================================================
+# Quellenaufbereitung – NUR metadata verwenden!
+# ============================================================
+
+def build_sources_metadata(docs: List) -> List[Dict[str, Any]]:
+    sources = []
+
+    for idx, d in enumerate(docs):
+        meta = d.metadata
+        snippet = d.page_content[:300].replace("\n", " ")
+
+        # PDF
+        if meta.get("type") == "pdf":
+            sources.append({
+                "id": idx + 1,
+                "source": "Prüfungsordnung (PDF)",
+                "page": meta.get("page"),
+                "url": meta.get("pdf_url"),   # KHÔNG tạo lại!
+                "snippet": snippet,
+            })
+            continue
+
+        # Hochschulgesetz NRW
+        if meta.get("type") == "hg":
+            sources.append({
+                "id": idx + 1,
+                "source": "Hochschulgesetz NRW",
+                "page": None,
+                "url": meta.get("viewer_url"),  # KHÔNG tạo lại!
+                "snippet": snippet,
+            })
+            continue
+
+    return sources
+
+# ============================================================
+# Kontextaufbereitung
+# ============================================================
+
+def format_context(docs: List) -> str:
+    if not docs:
+        return "(Kein relevanter Kontext gefunden.)"
+
+    blocks = []
+
+    for i, d in enumerate(docs):
+        meta = d.metadata
+        doc_type = meta.get("type")
+
+        label = "Prüfungsordnung" if doc_type == "pdf" else "Hochschulgesetz NRW"
+
+        if doc_type == "pdf":
+            page = meta.get("page")
+            label += f", Seite {page+1}" if isinstance(page, int) else ""
+
+        blocks.append(
+            f"[KONTEXT {i+1}] ({label})\n{d.page_content[:MAX_CHARS]}"
+        )
+
+    return "\n\n".join(blocks)
+
+# -----------------------------
+# Systemprompt — verschärft
+# -----------------------------
+
+SYSTEM_PROMPT = """
+Du bist ein hochpräziser juristischer Chatbot für Prüfungsrecht
+mit Zugriff nur auf:
+- die Prüfungsordnung (als PDF) und
+- das Hochschulgesetz NRW (als HTML aus der offiziellen Druckversion).
+Strenge Regeln:
+1. Antworte ausschließlich anhand des bereitgestellten Kontextes
+   (KONTEXT-Abschnitte). Wenn die Information nicht im Kontext steht,
+   sage ausdrücklich, dass dies aus den vorliegenden Dokumenten nicht
+   hervorgeht und du dazu nichts Sicheres sagen kannst.
+2. 
+   Keine Spekulationen, keine Vermutungen.
+3. Antworte in zusammenhängenden, ganzen Sätzen. Verwende keine Mischung aus Deutsch und Englisch.
+4. Nenne, soweit aus dem Kontext erkennbar,
+   - die rechtliche Grundlage (z.B. Paragraph, Artikel),
+   - das Dokument (Prüfungsordnung / Hochschulgesetz NRW),
+   - die Seite (bei der Prüfungsordnung), wenn im Kontext vorhanden.
+5. Füge KEINE externen Informationen hinzu, z.B. aus anderen Gesetzen,
+   Webseiten oder allgemeinem Wissen. Nur das, was im Kontext steht,
+   darf in der Antwort verwendet werden.
+Wenn der Kontext keine eindeutige Antwort zulässt, erkläre klar,
+warum keine sichere Antwort möglich ist und welche Informationen
+im Dokument fehlen.
+"""
+
+# -----------------------------
+# Hauptfunktion
+# -----------------------------
+
+def answer(question: str, retriever, chat_model) -> Tuple[str, List[Dict[str, Any]]]:
+    """
+    Haupt-RAG-Funktion:
+    - ruft retriever.invoke(question) auf,
+    - baut einen präzisen Prompt mit KONTEXT,
+    - ruft LLM auf,
+    - gibt Antworttext + Quellenliste zurück.
+    """
+    # 1. Dokumente holen
+    docs = retriever.invoke(question)
+    context_str = format_context(docs)
+
+    # 2. Prompt bauen
+    user_prompt = f"""
+FRAGE:
+{question}
+NUTZE AUSSCHLIESSLICH DIESEN KONTEXT:
+{context_str}
+AUFGABE:
+Formuliere eine juristisch korrekte, gut verständliche Antwort
+ausschließlich anhand des obigen Kontextes.
+- Wenn der Kontext aus den Dokumenten eine klare Antwort erlaubt,
+  erläutere diese strukturiert und in vollständigen Sätzen.
+- Wenn der Kontext KEINE klare Antwort erlaubt oder wichtige Informationen
+  fehlen, erkläre das offen und formuliere KEINE Vermutung.
+"""
+
+    msgs = [
+        SystemMessage(content=SYSTEM_PROMPT),
+        HumanMessage(content=user_prompt),
+    ]
+
+    # 3. LLM aufrufen
+    result = chat_model.invoke(msgs)
+    answer_text = result.content.strip()
+
+    # 4. Quellenliste bauen
+    sources = build_sources_metadata(docs)
+
+    return answer_text, sources

requirements.txt

@@ -0,0 +1,39 @@
+# === UI ===
+gradio
+gradio_pdf
+
+# === Supabase ===
+supabase
+postgrest
+httpx
+python-dotenv
+
+# === LangChain Core ===
+langchain
+langchain-community
+langchain-text-splitters
+langchain-openai
+huggingface-hub
+
+# === VectorStore ===
+faiss-cpu
+
+# === PDF + HTTP + HTML ===
+pypdf
+requests
+beautifulsoup4
+lxml
+
+# === Audio (STT/TTS local) ===
+transformers
+accelerate
+soundfile
+scipy
+numpy
+torchaudio
+torch
+librosa
+
+# === OpenAI + HF Hub ===
+openai
+huggingface_hub

retriever.py

@@ -0,0 +1,46 @@
+"""
+BƯỚC 5: RETRIEVER
+-----------------
+Tạo LangChain Retriever từ FAISS VectorStore.
+Retriever sẽ dùng trong bước RAG sau này:
+- retriever.get_relevant_documents(query)
+"""
+
+from langchain_community.vectorstores import FAISS
+
+# số chunk sẽ lấy cho mỗi câu hỏi
+RETRIEVER_K = 4
+
+def get_retriever(vectorstore: FAISS, k: int = RETRIEVER_K):
+    """
+    Tạo retriever từ FAISS VectorStore.
+    """
+    print(f">>> Creating retriever with k={k} ...")
+    retriever = vectorstore.as_retriever(search_kwargs={"k": k})
+    print(">>> Retriever ready.\n")
+    return retriever
+
+if __name__ == "__main__":
+    # Test: load -> split -> FAISS -> retriever.get_relevant_documents()
+    from load_documents import load_documents
+    from split_documents import split_documents
+    from vectorstore import build_vectorstore
+
+    print("=== TEST: retriever.get_relevant_documents ===\n")
+
+    docs = load_documents()
+    chunks = split_documents(docs)
+    vs = build_vectorstore(chunks)
+    retriever = get_retriever(vs, k=4)
+
+    query = "Wie lange habe ich Zeit, eine Prüfungsleistung zu wiederholen?"
+    print("Test query:")
+    print(" ", query, "\n")
+
+    retrieved_docs = retriever.invoke(query)
+
+    print(f"Retriever returned {len(retrieved_docs)} documents.")
+    for i, d in enumerate(retrieved_docs, start=1):
+        print(f"\n=== DOC {i} ===")
+        print(d.page_content[:400], "...")
+        print("Metadata:", d.metadata)

speech_io.py

@@ -0,0 +1,102 @@
+import numpy as np
+import soundfile as sf
+import librosa
+from transformers import pipeline
+
+ASR_MODEL_ID = "openai/whisper-small"   # multilingual
+TTS_MODEL_ID = "facebook/mms-tts-deu"  # bạn có thể thay nếu muốn đa ngôn ngữ
+
+_asr = None
+_tts = None
+
+
+# ============================================
+# LOAD AUDIO – chuẩn hóa 16kHz mono
+# ============================================
+def load_audio_16k(path):
+    audio, sr = sf.read(path)
+
+    # Stereo → Mono
+    if audio.ndim > 1:
+        audio = audio.mean(axis=1)
+
+    # Resample → 16kHz
+    if sr != 16000:
+        audio = librosa.resample(audio, orig_sr=sr, target_sr=16000)
+        sr = 16000
+
+    return audio.astype(np.float32), sr
+
+
+# ============================================
+# LOAD WHISPER PIPELINE (multilingual)
+# ============================================
+def get_asr_pipeline():
+    global _asr
+    if _asr is None:
+        _asr = pipeline(
+            task="automatic-speech-recognition",
+            model=ASR_MODEL_ID,
+            return_timestamps=False,
+            chunk_length_s=30,
+        )
+    return _asr
+
+
+# ============================================
+# MULTILINGUAL STT
+# ============================================
+def transcribe_audio(audio_path: str) -> str:
+    if audio_path is None:
+        return ""
+
+    audio, sr = load_audio_16k(audio_path)
+
+    # Nếu quá ngắn → Whisper sẽ sinh ký tự rác
+    if len(audio) < sr * 0.4:  
+        return ""
+
+    asr = get_asr_pipeline()
+
+    # Không đặt language → Whisper tự detect ngôn ngữ
+    result = asr(
+        {"array": audio, "sampling_rate": sr},
+        generate_kwargs={
+            "task": "transcribe",     # không translate — giữ nguyên ngôn ngữ gốc
+            "temperature": 0.0        # giảm hallucination như "ვვვ..."
+        }
+    )
+
+    text = result.get("text", "").strip()
+
+    # Fix edge case: nếu Whisper trả về ký tự vô nghĩa → bỏ qua
+    if set(text) <= {"ვ", " "}:
+        return ""
+
+    return text
+
+
+# ============================================
+# TEXT → SPEECH (chưa multilingual)
+# ============================================
+def get_tts_pipeline():
+    global _tts
+    if _tts is None:
+        _tts = pipeline(task="text-to-speech", model=TTS_MODEL_ID)
+    return _tts
+
+
+def synthesize_speech(text: str):
+    if not text.strip():
+        return None
+
+    tts = get_tts_pipeline()
+    out = tts(text)
+
+    audio = np.array(out["audio"], dtype=np.float32)
+    sr = out.get("sampling_rate", 16000)
+
+    max_val = np.max(np.abs(audio)) or 1.0
+    audio = audio / max_val
+
+    return sr, (audio * 32767).astype(np.int16)

split_documents.py

@@ -0,0 +1,27 @@
+# split_documents.py – v2
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+CHUNK_SIZE = 1500
+CHUNK_OVERLAP = 200
+
+def split_documents(docs):
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
+    chunks = splitter.split_documents(docs)
+
+    for c in chunks:
+        c.metadata["chunk_size"] = CHUNK_SIZE
+        c.metadata["chunk_overlap"] = CHUNK_OVERLAP
+
+    return chunks
+
+if __name__ == "__main__":
+    from load_documents import load_documents
+    docs = load_documents()
+    chunks = split_documents(docs)
+    print("Docs:", len(docs), "Chunks:", len(chunks))
+    print(chunks[0].page_content[:300], chunks[0].metadata)

upload_weblink_to_supabase.py

@@ -0,0 +1,130 @@
+import os
+import requests
+from bs4 import BeautifulSoup
+from supabase import create_client
+from dotenv import load_dotenv
+
+load_dotenv()
+
+SUPABASE_URL = os.environ["SUPABASE_URL"]
+SUPABASE_SERVICE_ROLE = os.environ["SUPABASE_SERVICE_ROLE"]
+
+supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
+
+# URL CHÍNH THỨC – không dùng Druckversion
+LAW_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
+
+def extract_paragraphs():
+    """
+    Lädt die aktuelle Fassung des Hochschulgesetzes NRW
+    von recht.nrw.de (br_text_anzeigen) und extrahiert Paragraphen.
+    Ergebnis: Liste von Dicts mit:
+      - abs_id:    para_1, para_2, ...
+      - title:     "§ 1 ...", "§ 2 ..."
+      - content:   gesamter Text des Paragraphen
+      - order_index: laufende Nummer
+    """
+    print(">>> Lade offizielles Hochschulgesetz NRW von recht.nrw.de …")
+
+
+    # html = requests.get(LAW_URL, timeout=30).text
+    # soup = BeautifulSoup(html, "html.parser")
+    # 5.12_2:13
+    resp = requests.get(LAW_URL, timeout=30)
+    resp.raise_for_status()
+    soup = BeautifulSoup(resp.text, "html.parser")
+
+    # 5.12_2:13
+    # Paragraph-Überschriften: häufig in <p>, <b> oder <strong>
+    candidates = soup.find_all(["p", "b", "strong"])
+
+
+    # Tất cả tiêu đề Paragraph xuất hiện trong <h2> hoặc <h3>
+    headers = soup.find_all(["h2", "h3"])
+
+    paragraphs = []
+    order = 1
+
+    # for header in headers:
+    #     title = header.get_text(" ", strip=True)
+
+    #     if not title.startswith("§"):
+    #         continue  # bỏ các h2/h3 không phải Paragraph
+
+    #     # Gom toàn bộ nội dung từ header đến trước h2/h3 tiếp theo
+    #     content_parts = []
+    #     sibling = header.find_next_sibling()
+
+    #     while sibling and sibling.name not in ["h2", "h3"]:
+    #         text = sibling.get_text(" ", strip=True)
+    #         if text:
+    #             content_parts.append(text)
+    #         sibling = sibling.find_next_sibling()
+
+    #     full_content = "\n".join(content_parts).strip()
+
+    #     para_id = f"para_{order}"
+
+    #     paragraphs.append({
+    #         "abs_id": para_id,
+    #         "title": title,
+    #         "content": full_content,
+    #         "order_index": order
+    #     })
+
+    #     order += 1
+
+    # print(f"✔ Extracted {len(paragraphs)} paragraphs (§).")
+    # return paragraphs
+    # 5.12_2:13
+    for tag in candidates:
+        text = tag.get_text(" ", strip=True)
+        if not text.startswith("§"):
+            continue
+
+        title = text
+        content_parts = []
+        sibling = tag.find_next_sibling()
+
+        while sibling and not (
+            (sibling.name in ["p", "b", "strong"])
+            and sibling.get_text(" ", strip=True).startswith("§")
+        ):
+            txt = sibling.get_text(" ", strip=True)
+            if txt:
+                content_parts.append(txt)
+            sibling = sibling.find_next_sibling()
+
+        full_content = "\n".join(content_parts).strip()
+        abs_id = f"para_{order}"
+
+        paragraphs.append(
+            {
+                "abs_id": abs_id,
+                "title": title,
+                "content": full_content,
+                "order_index": order,
+            }
+        )
+        order += 1
+
+    print(f"✔ {len(paragraphs)} Paragraphen extrahiert.")
+    return paragraphs
+
+def upload_to_supabase():
+    paras = extract_paragraphs()
+
+    print(">>> Leere Tabelle hg_nrw …")
+    supabase.table("hg_nrw").delete().neq("abs_id", "").execute()
+
+    print(">>> Upload nach Supabase …")
+    BATCH = 100
+    for i in range(0, len(paras), BATCH):
+        batch = paras[i:i+BATCH]
+        print(f"   - Upload batch {i} – {i+len(batch)-1}")
+        supabase.table("hg_nrw").upsert(batch).execute()
+
+    print("✔ DONE uploading complete NRW law.")
+
+if __name__ == "__main__":
+    upload_to_supabase()

vectorstore.py

@@ -0,0 +1,56 @@
+"""
+BƯỚC 4: VECTORSTORE (FAISS in-memory)
+-------------------------------------
+Tạo FAISS index từ các CHUNK văn bản.
+- Không ghi file .faiss nào, tất cả nằm trong RAM.
+- Embeddings được lấy từ get_embeddings() (Bước 3).
+"""
+
+from langchain_community.vectorstores import FAISS
+from embeddings import get_embeddings
+
+def build_vectorstore(chunks):
+    """
+    Nhận danh sách Document (đã split) và trả về FAISS VectorStore.
+    """
+    print(">>> Initialising embedding model for FAISS index ...")
+    embeddings = get_embeddings()
+
+    print(f">>> Building FAISS index from {len(chunks)} chunks ...")
+    vs = FAISS.from_documents(chunks, embeddings)
+    print(">>> FAISS index built.\n")
+    return vs
+
+if __name__ == "__main__":
+    # Test toàn pipeline: load -> split -> FAISS -> similarity_search
+    from load_documents import load_documents
+    from split_documents import split_documents
+
+    print("=== TEST: load_documents -> split_documents -> FAISS.similarity_search ===\n")
+
+    # 1) Load tài liệu (PDF + HTML) từ HuggingFace
+    docs = load_documents()
+
+    # 2) Split thành chunks
+    from pprint import pprint
+    print(f"Loaded {len(docs)} raw documents.")
+    chunks = split_documents(docs)
+    print(f"Split into {len(chunks)} chunks.\n")
+
+    # 3) Xây FAISS vectorstore
+    vectorstore = build_vectorstore(chunks)
+
+    # 4) Test similarity_search
+    query = "Fristen für die Prüfungsanmeldung im Bachelorstudium"
+    print("Test query:")
+    print(" ", query, "\n")
+
+    results = vectorstore.similarity_search(query, k=3)
+
+    print("Top-3 ähnliche Chunks aus dem VectorStore:")
+    for i, doc in enumerate(results, start=1):
+        print(f"\n=== RESULT {i} ===")
+        print(doc.page_content[:400], "...")
+        print("Metadata:", doc.metadata)
+
+