update brokens

app.py

@@ -5,7 +5,7 @@ import os
 
 from ui import (
     blog,
-    general_blog,
+    #general_blog,
     meta,
     twitter,
     price_matching
@@ -13,7 +13,7 @@ from ui import (
 
 from server import (
     blog as blog_srv,
-    general_blog as general_blog_srv,
+    #general_blog as general_blog_srv,
     meta as meta_srv,
     twitter as twitter_srv,
     price_matching as price_matching_srv
@@ -23,7 +23,7 @@ from server import (
 ui = ui.page_fluid(
     ui.page_navbar(
         blog.ui,
-        general_blog.ui,
+        #general_blog.ui,
         meta.ui,
         twitter.ui,
         price_matching.ui,
@@ -36,7 +36,7 @@ ui = ui.page_fluid(
 
 def server(input, output, session):
     blog_srv.server(input, output, session)
-    general_blog_srv.server(input, output, session)
+    #general_blog_srv.server(input, output, session)
     meta_srv.server(input, output, session)
     twitter_srv.server(input, output, session)
     price_matching_srv.server(input, output, session)

server/general_blog.py

@@ -17,29 +17,27 @@ SHOPIFY_API_VERSION = "2024-04"
 BLOG_ID = "73667707064"
 
 # === Static scraper for pokemon.com ===
-def scrape_section_content_from_url(url: str) -> str:
+async def scrape_section_content_from_url(url: str) -> str:
     try:
-        resp = requests.get(url, timeout=10, headers={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
-            "Accept-Language": "en-US,en;q=0.9",
-            "Referer": "https://www.google.com/"
-        })
-        if not resp.ok:
-            print(f"[ERROR] Request failed: {resp.status_code}")
-            return ""
-
-        soup = BeautifulSoup(resp.text, "html.parser")
-
-        # Match all divs that contain the full class string
-        content_blocks = soup.find_all("div")
-
-        if not content_blocks:
-            print("[WARN] No content blocks matched.")
-            return ""
-
-        texts = [div.get_text(separator=" ", strip=True) for div in content_blocks]
-        print(f"[INFO] Extracted {len(texts)} content blocks.")
-        return "\n\n".join(texts)
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            page = await browser.new_page()
+            await page.goto(url, timeout=30000)
+            await page.wait_for_load_state("networkidle")
+            html = await page.content()
+            await browser.close()
+
+            soup = BeautifulSoup(html, "html.parser")
+
+            # Match all divs and extract text
+            content_blocks = soup.find_all("div")
+            if not content_blocks:
+                print("[WARN] No <div> elements found.")
+                return ""
+
+            texts = [div.get_text(separator=" ", strip=True) for div in content_blocks if div.get_text(strip=True)]
+            print(f"[INFO] Extracted {len(texts)} content blocks.")
+            return "\n\n".join(texts)
 
     except Exception as e:
         print(f"[ERROR] Scraping failed: {e}")