commit

load_documents.py

@@ -1,128 +1,131 @@
 """
-LOAD_DOCUMENTS – SINGLE SOURCE OF TRUTH
-Nhiệm vụ:
-1) Lade Prüfungsordnung PDF direkt aus Supabase-Storage.
-2) Lade Hochschulgesetz NRW aus Supabase-Tabelle hg_nrw.
-3) Cung cấp metadata đầy đủ để các file khác KHÔNG PHẢI tính lại URL.
+BƯỚC 1: LOAD DOCUMENTS
+-----------------------
+Debug-full version
+
+- Lädt Prüfungsordnung (PDF) seitenweise.
+- Lädt Hochschulgesetz NRW aus dem im Dataset gespeicherten HTML,
+  und zerlegt es in einzelne Absätze (Document pro <p>).
 """
 
-import os
-import tempfile
-from dotenv import load_dotenv
+from huggingface_hub import hf_hub_download, list_repo_files
 from langchain_community.document_loaders import PyPDFLoader
 from langchain_core.documents import Document
-from supabase import create_client
-
-load_dotenv()
-
-import urllib.parse
-
-# ===== Supabase config =====
-SUPABASE_URL = os.getenv("SUPABASE_URL")
-SUPABASE_SERVICE_ROLE = os.getenv("SUPABASE_SERVICE_ROLE")
-
-supabase = create_client(SUPABASE_URL, SUPABASE_SERVICE_ROLE)
-
-# ===== Storage Config =====
+from bs4 import BeautifulSoup
 
-#import urllib.parse
+DATASET = "Nguyen5/docs"
 PDF_FILE = "f10_bpo_ifb_tei_mif_wii_2021-01-04.pdf"
+HTML_FILE = "Hochschulgesetz_NRW.html"  # konsistent mit hg_nrw.py
+
+def _load_hg_paragraph_documents(html_path: str):
+    """
+    Liest das generierte Hochschulgesetz-HTML ein und erzeugt
+    pro <p>-Element einen LangChain-Document mit:
+      - page_content = Text des Absatzes
+      - metadata:
+          source       = "Hochschulgesetz NRW (HTML)"
+          filename     = HTML_FILE
+          paragraph_id = id-Attribut (z.B. 'hg_abs_12'), falls vorhanden
+    """
+    with open(html_path, "r", encoding="utf-8") as f:
+        html = f.read()
+
+    soup = BeautifulSoup(html, "html.parser")
+    docs = []
 
-PDF_BUCKET = "File PDF"
-ENC_BUCKET = urllib.parse.quote(PDF_BUCKET)   # "File%20PDF"
-
-#PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{PDF_BUCKET}/{PDF_FILE}"
-PDF_URL = f"{SUPABASE_URL}/storage/v1/object/public/{ENC_BUCKET}/{PDF_FILE}"
-
-
-# ===== Viewer URL =====
-HG_VIEWER_BUCKET = "hg_viewer"
-HG_VIEWER_FILE = "hg_clean.html"
-HG_VIEWER_URL = f"{SUPABASE_URL}/storage/v1/object/public/{HG_VIEWER_BUCKET}/{HG_VIEWER_FILE}"
-
-
-# ============================================================
-# 1) PDF aus Supabase laden
-# ============================================================
-
-def load_pdf_from_supabase() -> list[Document]:
-    print("📥 Lade Prüfungsordnung PDF aus Supabase...")
-
-    response = supabase.storage.from_(PDF_BUCKET).download(PDF_FILE)
-    if response is None:
-        raise ValueError("❌ Konnte PDF nicht laden!")
-
-    # Temporäre Datei
-    with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
-        tmp.write(response)
-        temp_pdf_path = tmp.name
+    for p in soup.find_all("p"):
+        text = p.get_text(" ", strip=True)
+        if not text:
+            continue
 
-    pages = PyPDFLoader(temp_pdf_path).load()
+        pid = p.get("id")
 
-    for i, p in enumerate(pages):
-        p.metadata = {
-            "type": "pdf",
-            "source": "Prüfungsordnung",
-            "page": i,
-            "pdf_url": f"{PDF_URL}#page={i}",
-            "filename": PDF_FILE,
+        metadata = {
+            "source": "Hochschulgesetz NRW (HTML)",
+            "filename": HTML_FILE,
         }
+        if pid:
+            metadata["paragraph_id"] = pid
 
-    print(f"✔ {len(pages)} PDF-Seiten geladen.")
-    return pages
+        docs.append(Document(page_content=text, metadata=metadata))
 
+    print(f"Loaded {len(docs)} paragraph Documents from HG-HTML.\n")
+    return docs
 
-# ============================================================
-# 2) HG aus Tabelle laden
-# ============================================================
+def load_documents():
+    print("=== START: load_documents() ===\n")
 
-def load_hg_from_supabase() -> list[Document]:
-    print("📥 Lade Hochschulgesetz NRW aus Tabelle hg_nrw...")
+    # -------------------------
+    # Check files in dataset
+    # -------------------------
+    print(">>> Checking dataset file list from HuggingFace...")
+    files = list_repo_files(DATASET, repo_type="dataset")
+    print("Files in dataset:", files, "\n")
 
-    res = (
-        supabase.table("hg_nrw")
-        .select("*")
-        .order("order_index", desc=False)
-        .execute()
-    )
-    rows = res.data or []
     docs = []
 
-    for row in rows:
-        abs_id = row["abs_id"]
-        title = row["title"]
-        content = row["content"]
-
-        viewer_url = f"{HG_VIEWER_URL}#{abs_id}"
-
-        docs.append(
-            Document(
-                page_content=content,
-                metadata={
-                    "type": "hg",
-                    "source": "Hochschulgesetz NRW",
-                    "abs_id": abs_id,
-                    "title": title,
-                    "viewer_url": viewer_url,
-                },
-            )
+    # -------------------------
+    # Load PDF
+    # -------------------------
+    print(">>> Step 1: Download PDF from HuggingFace...")
+    try:
+        pdf_path = hf_hub_download(
+            repo_id=DATASET,
+            filename=PDF_FILE,
+            repo_type="dataset",
         )
+        print(f"Downloaded PDF to local cache:\n{pdf_path}\n")
+    except Exception as e:
+        print("ERROR downloading PDF:", e)
+        return []
+
+    print(">>> Step 1.1: Loading PDF pages...")
+    try:
+        pdf_docs = PyPDFLoader(pdf_path).load()
+        print(f"Loaded {len(pdf_docs)} PDF pages.\n")
+    except Exception as e:
+        print("ERROR loading PDF:", e)
+        return []
+
+    for d in pdf_docs:
+        d.metadata["source"] = "Prüfungsordnung (PDF)"
+        d.metadata["filename"] = PDF_FILE
+
+    docs.extend(pdf_docs)
+
+    # -------------------------
+    # Load HTML (Hochschulgesetz NRW)
+    # -------------------------
+    print(">>> Step 2: Download HTML from HuggingFace...")
+    try:
+        html_path = hf_hub_download(
+            repo_id=DATASET,
+            filename=HTML_FILE,
+            repo_type="dataset",
+        )
+        print(f"Downloaded HTML to local cache:\n{html_path}\n")
+    except Exception as e:
+        print("ERROR downloading HTML:", e)
+        return docs
 
-    print(f"✔ {len(docs)} HG-Absätze geladen.")
-    return docs
+    print(">>> Step 2.1: Loading HG-HTML and splitting into paragraphs...")
+    try:
+        html_docs = _load_hg_paragraph_documents(html_path)
+    except Exception as e:
+        print("ERROR loading / parsing HTML:", e)
+        return docs
 
+    docs.extend(html_docs)
 
-# ============================================================
-# 3) ALLES LADEN
-# ============================================================
+    print("=== DONE: load_documents() ===\n")
+    return docs
 
-def load_all_documents():
-    pdf_docs = load_pdf_from_supabase()
-    hg_docs = load_hg_from_supabase()
-    return pdf_docs + hg_docs
+if __name__ == "__main__":
+    print("\n=== Running load_documents.py directly ===\n")
+    docs = load_documents()
+    print(f"\n>>> TOTAL documents loaded: {len(docs)}")
 
+    if len(docs):
+        print("\nExample metadata from 1st document:")
+        print(docs[0].metadata)
 
-if __name__ == "__main__":
-    docs = load_all_documents()
-    print("📚 Gesamt:", len(docs))
-    print("🔎 Beispiel metadata:", docs[0].metadata)