commit

app.py

@@ -1,4 +1,4 @@
-# app.py – fixed Quelle links
+# app.py
 import os
 import base64
 import gradio as gr
@@ -9,22 +9,29 @@ from rag_pipeline import rag_answer
 
 client = OpenAI()
 BUCKET = os.environ["SUPABASE_BUCKET"]
-
 SUPABASE_URL = os.environ["SUPABASE_URL"]
-PDF_URL      = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
-HG_URL       = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/hochschulgesetz.html"
-
 
-def encode_pdf_src():
-    pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
-    return f"data:application/pdf;base64,{base64.b64encode(pdf_bytes).decode('utf-8')}"
+# ------------------------------------------
+# URLs cho Prüfungsordnung (PDF) + HG NRW
+# ------------------------------------------
 
+# PDF nằm trong Supabase (như trước)
+PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
 
-def encode_html():
-    html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
-    return html_bytes.decode("utf-8", errors="ignore")
+# ⚠️ Đây là link chính thức của Hochschulgesetz NRW trên recht.nrw.de
+HG_URL  = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
 
+# ------------------------------------------
+# Viewer PDF base64
+# ------------------------------------------
+def encode_pdf_src():
+    pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
+    b64 = base64.b64encode(pdf_bytes).decode("utf-8")
+    return f"data:application/pdf;base64,{b64}"
 
+# ------------------------------------------
+# Speech-to-text (Whisper)
+# ------------------------------------------
 def transcribe(audio_path):
     if audio_path is None:
         return ""
@@ -32,92 +39,111 @@ def transcribe(audio_path):
         result = client.audio.transcriptions.create(
             model="whisper-1",
             file=f,
-            language="de",
+            language="de",      # tiếng Đức
             temperature=0.0
         )
     return (result.text or "").strip()
 
-
+# ------------------------------------------
+# HÀM CHAT CHÍNH
+# ------------------------------------------
 def chat_fn(text, audio, history):
     text = (text or "").strip()
 
+    # 1) Ưu tiên TEXT; chỉ dùng audio nếu không có text
     if text:
         question = text
-    elif audio:
+    elif audio is not None:
         question = transcribe(audio)
     else:
-        return history, "<p>Bitte Text oder Mikrofon benutzen.</p>", None
+        return history, "", None  # không có input
 
     if not question:
-        return history, "<p>Spracherkennung fehlgeschlagen.</p>", None
+        return history, "Spracherkennung fehlgeschlagen.", None
 
+    # 2) Gọi RAG
     answer, docs = rag_answer(question, history or [])
 
-    html = "<ol>"
+    # 3) Xây block Quellen ở dạng Markdown, DÙNG META từ docs
+    quellen_md_lines = ["", "### 📚 Quellen (verwendete Dokumentstellen):"]
     for i, d in enumerate(docs):
-        meta = d["metadata"]
-        src  = meta.get("source")
-        page = meta.get("page")
-        anchor = meta.get("anchor_id")
-
-        # PDF vs HTML
-        if src == "Prüfungsordnung (PDF)":
-            link = f"{PDF_URL}#page={page+1}" if isinstance(page, int) else PDF_URL
+        meta = d.get("metadata", {}) or {}
+        src = meta.get("source", "?")
+        page = meta.get("page", None)
+        anchor_id = meta.get("anchor_id")
+
+        # Prüfungsordnung (PDF) – nhảy đúng Seite
+        if src.startswith("Prüfungsordnung"):
+            # trong ingest page lưu 1-based; nếu bạn dùng 0-based thì +1 ở đây
+            page_num = page if isinstance(page, int) else None
+            if page_num:
+                url = f"{PDF_URL}#page={page_num}"
+                title = f"Quelle {i+1} – {src}, Seite {page_num}"
+            else:
+                url = PDF_URL
+                title = f"Quelle {i+1} – {src}"
+        # Hochschulgesetz NRW – dùng URL chính thức + anchor_id (para)
         else:
-            link = f"{HG_URL}#{anchor}" if anchor else HG_URL
+            if anchor_id:
+                url = f"{HG_URL}#{anchor_id}"
+            else:
+                url = HG_URL
+            title = f"Quelle {i+1} – Hochschulgesetz NRW"
+
+        snippet = (d.get("content") or "")[:200].replace("\n", " ")
 
-        snippet = d["content"][:200].replace("\n", " ")
+        quellen_md_lines.append(f"- [{title}]({url})")
+        quellen_md_lines.append(f"  > {snippet}")
 
-        html += f"""
-        <li>
-            <a href="{link}" target="_blank">
-                <b>Quelle {i+1}: {src}</b>
-            </a><br>
-            {snippet}...
-        </li>
-        """
-    html += "</ol>"
+    quellen_md = "\n".join(quellen_md_lines)
+
+    # 4) GỘP câu trả lời + Quellen vào NỘI DUNG CHATBOT
+    bot_msg = answer + "\n\n" + quellen_md
 
     new_history = (history or []) + [
         {"role": "user", "content": question},
-        {"role": "assistant", "content": answer},
+        {"role": "assistant", "content": bot_msg},
     ]
 
-    return new_history, html, gr.update(value=None)
-
+    # Trả về history (hiển thị trong Chatbot) + block Markdown (nếu muốn xem riêng) + reset audio
+    return new_history, bot_msg, gr.update(value=None)
 
+# ------------------------------------------
+# GIAO DIỆN
+# ------------------------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("# ⚖️ Prüfungsrechts-Chatbot (RAG mit Supabase)")
+    gr.Markdown("# ⚖️ Sprachbasierter Chatbot für Prüfungsrecht")
 
     with gr.Row():
-
+        # Bên trái: Chat
         with gr.Column(scale=3):
-            chatbot = gr.Chatbot()
+            # Chatbot RENDER Markdown (type="messages")
+            chatbot = gr.Chatbot(type="messages", label="Chat (RAG)", height=500)
             text_input = gr.Textbox(label="Frage eingeben")
-            audio_input = gr.Audio(
-                type="filepath", label="Spracheingabe (Mikrofon)"
-            )
+            audio_input = gr.Audio(type="filepath", label="Spracheingabe (Mikrofon)")
             send_btn = gr.Button("Senden")
 
+            # Preview Markdown của câu trả lời cuối (tuỳ chọn)
+            answer_preview = gr.Markdown("")
+
+        # Bên phải: Viewer
         with gr.Column(scale=2):
-            gr.Markdown("### Prüfungsordnung (PDF)")
+            gr.Markdown("### 📄 Prüfungsordnung (PDF)")
             gr.HTML(
-                f"<iframe src='{encode_pdf_src()}' width='100%' height='260px'></iframe>"
+                f"<iframe src='{encode_pdf_src()}' width='100%' height='250' style='border:none;'></iframe>"
             )
 
-            gr.Markdown("### Hochschulgesetz NRW")
+            gr.Markdown("### 📘 Hochschulgesetz NRW (offizielle Seite)")
             gr.HTML(
-                f"<div style='overflow:auto;height:260px;'>{encode_html()}</div>"
+                f"<iframe src='{HG_URL}' width='100%' height='250' style='border:none;'></iframe>"
             )
 
-            sources_html = gr.HTML()
-
+    # Nút gửi
     send_btn.click(
         chat_fn,
-        [text_input, audio_input, chatbot],
-        [chatbot, sources_html, audio_input]
+        inputs=[text_input, audio_input, chatbot],
+        outputs=[chatbot, answer_preview, audio_input],
     )
 
 if __name__ == "__main__":
-    demo.launch(ssr_mode=False)
-
+    demo.queue().launch(ssr_mode=False)

ingest.py

@@ -1,128 +1,104 @@
-# app.py – Quelle clickable & styled viewer (recht.nrw.de)
-
+# ingest.py
 import os
-import base64
-import gradio as gr
-from openai import OpenAI
+from io import BytesIO
+from bs4 import BeautifulSoup
+from pypdf import PdfReader
 
-from supabase_client import load_file_bytes
-from rag_pipeline import rag_answer
+from supabase_client import supabase, load_file_bytes
+from langchain_openai import OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
 
-client = OpenAI()
 BUCKET = os.environ["SUPABASE_BUCKET"]
-
 SUPABASE_URL = os.environ["SUPABASE_URL"]
 
-# Prüfungsordnung PDF – Supabase public URL
 PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
+HG_URL  = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/hochschulgesetz.html"
 
-# Hochschulgesetz NRW – offizielle recht.nrw.de URL (WICHTIG)
-HG_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
 
-
-# ------------------------------------------
-# PDF inline viewer (Base64)
-# ------------------------------------------
-def encode_pdf_src():
+def load_pdf_docs():
     pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
-    b64 = base64.b64encode(pdf_bytes).decode("utf-8")
-    return f"data:application/pdf;base64,{b64}"
-
-
-# ------------------------------------------
-# Speech-to-text
-# ------------------------------------------
-def transcribe(audio_path):
-    if audio_path is None:
-        return ""
-    with open(audio_path, "rb") as f:
-        result = client.audio.transcriptions.create(
-            model="whisper-1",
-            file=f,
-            language="de",
-            temperature=0.0  
+    reader = PdfReader(BytesIO(pdf_bytes))
+
+    docs = []
+    for i, page in enumerate(reader.pages):
+        text = page.extract_text() or ""
+
+        docs.append(
+            Document(
+                page_content=text,
+                metadata={
+                    "source": "Prüfungsordnung (PDF)",
+                    "page": i,                  # ZERO-based: Seite = i+1
+                    "pdf_url": PDF_URL,         # Basis-URL
+                },
+            )
         )
-    return (result.text or "").strip()
+    return docs
 
 
-# ------------------------------------------
-# Chat logic with inline Quelle links
-# ------------------------------------------
-def chat_fn(text, audio, history):
-    text = (text or "").strip()
+def load_html_docs():
+    html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
+    html = html_bytes.decode("utf-8", errors="ignore")
 
-    if text:
-        question = text
-    elif audio:
-        question = transcribe(audio)
-    else:
-        return history, "<p>Bitte Text oder Mikrofon benutzen.</p>", None
+    soup = BeautifulSoup(html, "html.parser")
+    text = soup.get_text(separator="\n")
 
-    answer, docs = rag_answer(question, history or [])
-
-    # Quellenblock (NOW CLICKABLE IN CHAT)
-    quellen_md = "### 📚 Quellen\n"
-    for i, d in enumerate(docs):
-        meta = d["metadata"]
-        src = meta.get("source")
-
-        # PDF
-        if src.startswith("Prüfungsordnung"):
-            page = meta.get("page", 0) + 1
-            url = f"{PDF_URL}#page={page}"
-            title = f"Quelle {i+1}: {src}, Seite {page}"
+    # HTML nicht in Paragraphen getrennt → wir chunk’en später
+    return [
+        Document(
+            page_content=text,
+            metadata={
+                "source": "Hochschulgesetz NRW",
+                # anchor_id wird erst beim Chunken vergeben
+            },
+        )
+    ]
 
-        # HTML – Rechtsportal NRW (WICHTIG)
-        else:
-            anchor = meta.get("anchor_id")
-            url = f"{HG_URL}#{anchor}"
-            title = f"Quelle {i+1}: Hochschulgesetz NRW (§)"
 
-        snippet = d["content"][:120].replace("\n", " ")
+def chunk_docs(docs):
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=900,
+        chunk_overlap=100,
+    )
+    return splitter.split_documents(docs)
 
-        quellen_md += f"- [{title}]({url})\n  > {snippet}\n"
 
-    bot_msg = f"{answer}\n\n{quellen_md}"
+def ingest():
+    pdf_docs = load_pdf_docs()
+    hg_docs = load_html_docs()
 
-    new_history = (history or []) + [
-        {"role": "user", "content": question},
-        {"role": "assistant", "content": bot_msg},
-    ]
+    chunks = chunk_docs(pdf_docs + hg_docs)
 
-    return new_history, bot_msg, gr.update(value=None)
+    po_idx = 1
+    hg_idx = 1
 
+    for d in chunks:
+        src = d.metadata["source"]
 
-# ------------------------------------------
-# UI
-# ------------------------------------------
-with gr.Blocks() as demo:
-    gr.Markdown("# ⚖️ Prüfungsrechts-Chatbot (Quelle-clickable)")
+        if src == "Prüfungsordnung (PDF)":
+            d.metadata["anchor_id"] = f"po_{po_idx}"
+            po_idx += 1
+        else:
+            d.metadata["anchor_id"] = f"hg_{hg_idx}"
+            hg_idx += 1
 
-    with gr.Row():
+        # HTML Quelle als vollständige URL
+        if src == "Hochschulgesetz NRW":
+            d.metadata["url"] = f"{HG_URL}#{d.metadata['anchor_id']}"
 
-        with gr.Column(scale=2):
-            chatbot = gr.Chatbot(type="messages", height=550)
-            text_input = gr.Textbox(label="Frage")
-            audio_input = gr.Audio(type="filepath", label="Mikrofon")
-            send_btn = gr.Button("Senden")
-            answer_preview = gr.Markdown("")
+    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
 
-            send_btn.click(
-                chat_fn,
-                inputs=[text_input, audio_input, chatbot],
-                outputs=[chatbot, answer_preview, audio_input]
-            )
+    for d in chunks:
+        emb = embeddings.embed_query(d.page_content)
+        supabase.table("documents").insert({
+            "content": d.page_content,
+            "metadata": d.metadata,
+            "embedding": emb
+        }).execute()
 
-        with gr.Column(scale=1):
-            gr.Markdown("### Prüfungsordnung (PDF)")
-            gr.HTML(
-                f"<iframe src='{encode_pdf_src()}' width='100%' height='330px' style='border:none'></iframe>"
-            )
+    print("OK ✔ ingest xong – PDF + HTML mit Quelle-URL")
 
-            gr.Markdown("### Hochschulgesetz NRW (offizielle Seite)")
-            gr.HTML(
-                f"<iframe src='{HG_URL}' width='100%' height='330px' style='border:none'></iframe>"
-            )
 
 if __name__ == "__main__":
-    demo.launch(ssr_mode=False)
+    ingest()