commit

ingest.py

@@ -1,104 +1,128 @@
-# ingest.py
+# app.py – Quelle clickable & styled viewer (recht.nrw.de)
+
 import os
-from io import BytesIO
-from bs4 import BeautifulSoup
-from pypdf import PdfReader
+import base64
+import gradio as gr
+from openai import OpenAI
 
-from supabase_client import supabase, load_file_bytes
-from langchain_openai import OpenAIEmbeddings
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_core.documents import Document
+from supabase_client import load_file_bytes
+from rag_pipeline import rag_answer
 
+client = OpenAI()
 BUCKET = os.environ["SUPABASE_BUCKET"]
+
 SUPABASE_URL = os.environ["SUPABASE_URL"]
 
+# Prüfungsordnung PDF – Supabase public URL
 PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/pruefungsordnung.pdf"
-HG_URL  = f"{SUPABASE_URL}/storage/v1/object/public/{BUCKET}/hochschulgesetz.html"
 
+# Hochschulgesetz NRW – offizielle recht.nrw.de URL (WICHTIG)
+HG_URL = "https://recht.nrw.de/lmi/owa/br_text_anzeigen?v_id=10000000000000000654"
 
-def load_pdf_docs():
+
+# ------------------------------------------
+# PDF inline viewer (Base64)
+# ------------------------------------------
+def encode_pdf_src():
     pdf_bytes = load_file_bytes(BUCKET, "pruefungsordnung.pdf")
-    reader = PdfReader(BytesIO(pdf_bytes))
-
-    docs = []
-    for i, page in enumerate(reader.pages):
-        text = page.extract_text() or ""
-
-        docs.append(
-            Document(
-                page_content=text,
-                metadata={
-                    "source": "Prüfungsordnung (PDF)",
-                    "page": i,                  # ZERO-based: Seite = i+1
-                    "pdf_url": PDF_URL,         # Basis-URL
-                },
-            )
+    b64 = base64.b64encode(pdf_bytes).decode("utf-8")
+    return f"data:application/pdf;base64,{b64}"
+
+
+# ------------------------------------------
+# Speech-to-text
+# ------------------------------------------
+def transcribe(audio_path):
+    if audio_path is None:
+        return ""
+    with open(audio_path, "rb") as f:
+        result = client.audio.transcriptions.create(
+            model="whisper-1",
+            file=f,
+            language="de",
+            temperature=0.0  
         )
-    return docs
+    return (result.text or "").strip()
 
 
-def load_html_docs():
-    html_bytes = load_file_bytes(BUCKET, "hochschulgesetz.html")
-    html = html_bytes.decode("utf-8", errors="ignore")
+# ------------------------------------------
+# Chat logic with inline Quelle links
+# ------------------------------------------
+def chat_fn(text, audio, history):
+    text = (text or "").strip()
 
-    soup = BeautifulSoup(html, "html.parser")
-    text = soup.get_text(separator="\n")
+    if text:
+        question = text
+    elif audio:
+        question = transcribe(audio)
+    else:
+        return history, "<p>Bitte Text oder Mikrofon benutzen.</p>", None
 
-    # HTML nicht in Paragraphen getrennt → wir chunk’en später
-    return [
-        Document(
-            page_content=text,
-            metadata={
-                "source": "Hochschulgesetz NRW",
-                # anchor_id wird erst beim Chunken vergeben
-            },
-        )
-    ]
+    answer, docs = rag_answer(question, history or [])
+
+    # Quellenblock (NOW CLICKABLE IN CHAT)
+    quellen_md = "### 📚 Quellen\n"
+    for i, d in enumerate(docs):
+        meta = d["metadata"]
+        src = meta.get("source")
 
+        # PDF
+        if src.startswith("Prüfungsordnung"):
+            page = meta.get("page", 0) + 1
+            url = f"{PDF_URL}#page={page}"
+            title = f"Quelle {i+1}: {src}, Seite {page}"
 
-def chunk_docs(docs):
-    splitter = RecursiveCharacterTextSplitter(
-        chunk_size=900,
-        chunk_overlap=100,
-    )
-    return splitter.split_documents(docs)
+        # HTML – Rechtsportal NRW (WICHTIG)
+        else:
+            anchor = meta.get("anchor_id")
+            url = f"{HG_URL}#{anchor}"
+            title = f"Quelle {i+1}: Hochschulgesetz NRW (§)"
 
+        snippet = d["content"][:120].replace("\n", " ")
 
-def ingest():
-    pdf_docs = load_pdf_docs()
-    hg_docs = load_html_docs()
+        quellen_md += f"- [{title}]({url})\n  > {snippet}\n"
 
-    chunks = chunk_docs(pdf_docs + hg_docs)
+    bot_msg = f"{answer}\n\n{quellen_md}"
 
-    po_idx = 1
-    hg_idx = 1
+    new_history = (history or []) + [
+        {"role": "user", "content": question},
+        {"role": "assistant", "content": bot_msg},
+    ]
 
-    for d in chunks:
-        src = d.metadata["source"]
+    return new_history, bot_msg, gr.update(value=None)
 
-        if src == "Prüfungsordnung (PDF)":
-            d.metadata["anchor_id"] = f"po_{po_idx}"
-            po_idx += 1
-        else:
-            d.metadata["anchor_id"] = f"hg_{hg_idx}"
-            hg_idx += 1
 
-        # HTML Quelle als vollständige URL
-        if src == "Hochschulgesetz NRW":
-            d.metadata["url"] = f"{HG_URL}#{d.metadata['anchor_id']}"
+# ------------------------------------------
+# UI
+# ------------------------------------------
+with gr.Blocks() as demo:
+    gr.Markdown("# ⚖️ Prüfungsrechts-Chatbot (Quelle-clickable)")
 
-    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
+    with gr.Row():
 
-    for d in chunks:
-        emb = embeddings.embed_query(d.page_content)
-        supabase.table("documents").insert({
-            "content": d.page_content,
-            "metadata": d.metadata,
-            "embedding": emb
-        }).execute()
+        with gr.Column(scale=2):
+            chatbot = gr.Chatbot(type="messages", height=550)
+            text_input = gr.Textbox(label="Frage")
+            audio_input = gr.Audio(type="filepath", label="Mikrofon")
+            send_btn = gr.Button("Senden")
+            answer_preview = gr.Markdown("")
 
-    print("OK ✔ ingest xong – PDF + HTML mit Quelle-URL")
+            send_btn.click(
+                chat_fn,
+                inputs=[text_input, audio_input, chatbot],
+                outputs=[chatbot, answer_preview, audio_input]
+            )
 
+        with gr.Column(scale=1):
+            gr.Markdown("### Prüfungsordnung (PDF)")
+            gr.HTML(
+                f"<iframe src='{encode_pdf_src()}' width='100%' height='330px' style='border:none'></iframe>"
+            )
+
+            gr.Markdown("### Hochschulgesetz NRW (offizielle Seite)")
+            gr.HTML(
+                f"<iframe src='{HG_URL}' width='100%' height='330px' style='border:none'></iframe>"
+            )
 
 if __name__ == "__main__":
-    ingest()
+    demo.launch(ssr_mode=False)