scarping update

main.py

@@ -44,18 +44,7 @@ OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 init_openai(OPENAI_API_KEY)
 
 
-# ------------------ HELPER ------------------
-def process_url(url: str):
-    driver = init_driver()
-    try:
-        raw_text = fetch_page_text(driver, url)
-        cleaned_text = clean_text_with_openai(raw_text)
-        pdf_file = url_to_filename(url, OUTPUT_FOLDER)
-        save_to_pdf(cleaned_text, pdf_file)
-        return str(pdf_file)
-    finally:
-        driver.quit()
-        
+# ------------------ HELPER ------------------        
 def clear_documents_folder():
     """Hapus semua file di folder documents"""
     for file in OUTPUT_FOLDER.iterdir():
@@ -82,32 +71,47 @@ async def build_vector_db():
 
 
 @app.post("/web-scraping/")
-async def scrape_urls(urls: str = Form(...)):
+async def scrape_urls(filename: str = Form(...), urls: str = Form(...), select: str = "huggingface"):
     """
-    Terima daftar URL dipisah koma, setiap URL menjadi 1 PDF.
-    Sebelum memulai, hapus semua file di folder documents.
+    Ambil semua URL, gabungkan teks → bersihkan → satu PDF utuh
     """
     # Hapus semua file lama
     clear_documents_folder()
 
     # Parse URL
     url_list = [u.strip() for u in urls.split(",") if u.strip()]
-    pdf_files = [None] * len(url_list)
-
+    extracted_texts = [None] * len(url_list)
     threads = []
 
-    def worker(i, url):
-        pdf_files[i] = process_url(url)
+    # STEP 1: Ekstraksi halaman
+    def worker_extract(i, url):
+        driver = init_driver_local() if select == "local" else init_driver()
+        try:
+            extracted_texts[i] = fetch_page_text(driver, url)
+        finally:
+            driver.quit()
 
     for i, url in enumerate(url_list):
-        t = threading.Thread(target=worker, args=(i, url))
+        t = threading.Thread(target=worker_extract, args=(i, url))
         t.start()
         threads.append(t)
 
     for t in threads:
         t.join()
 
-    return JSONResponse({"success": True, "pdf_files": pdf_files})
+    # STEP 2: Gabungkan semua teks
+    combined_text = ""
+    for url, text in zip(url_list, extracted_texts):
+        combined_text += f"===== URL: {url} =====\n{text}\n\n"
+
+    # STEP 3: Bersihkan dengan OpenAI
+    cleaned_text = clean_text_with_openai(combined_text)
+
+    # STEP 4: Simpan PDF
+    output_file = OUTPUT_FOLDER / f"{filename}.pdf"
+    save_to_pdf(cleaned_text, output_file)
+
+    return JSONResponse({"success": True, "pdf_file": str(output_file)})
     
 
 @app.post("/ask")

utils.py

@@ -1,6 +1,3 @@
-from selenium.webdriver.chrome.service import Service
-from selenium.webdriver.chrome.options import Options
-from selenium import webdriver
 from bs4 import BeautifulSoup
 from pathlib import Path
 from fpdf import FPDF
@@ -60,6 +57,10 @@ logging.basicConfig(
 
 # ---------------- SELENIUM ----------------
 def init_driver(headless=True):
+    from selenium.webdriver.chrome.service import Service
+    from selenium.webdriver.chrome.options import Options
+    from selenium import webdriver
+    
     options = Options()
     if headless:
         options.add_argument("--headless=new")
@@ -73,6 +74,20 @@ def init_driver(headless=True):
     logging.info("WebDriver berhasil diinisialisasi")
     return driver
 
+
+def init_driver_local(headless=True):
+    from selenium import webdriver
+    from selenium.webdriver.chrome.options import Options
+    
+    options = Options()
+    if headless:
+        options.add_argument("--headless")
+    options.add_argument("--disable-gpu")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--window-size=1920,1080")
+    driver = webdriver.Chrome(options=options)
+    return driver
+
 def fetch_page_text(driver, url: str) -> str:
     logging.info("Mengambil halaman: %s", url)
     driver.get(url)