Update app.py

app.py

@@ -8,7 +8,9 @@ from playwright.async_api import async_playwright
 from bs4 import BeautifulSoup
 import pandas as pd
 import time
-import asyncio
+import random
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from Gradio_UI import GradioUI
 
@@ -25,65 +27,78 @@ def scrape_drug_reviews_tool(drug_name: str, max_pages: int = 3) -> dict :
     Output: a dictionary url:review mapping the url of a review to the text of the review
     """
     try:
-        df = asyncio.run(scrape_drugs_com_reviews_playwright(drug_name, max_pages))
+        df = asyncio.run(scrape_drugs_com_reviews_requests(drug_name, max_pages))
         return df.to_dict(orient="records")
     except Exception as e:
         return {"error": str(e)}
 
 
         
-async def scrape_drugs_com_reviews_playwright(drug_name, max_pages=3, delay=2):
+# List of User-Agents for rotation
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:115.0) Gecko/20100101 Firefox/115.0"
+]
+
+# Retry logic wrapper
+def requests_retry_session(retries=3, backoff_factor=0.5, status_forcelist=(500, 502, 503, 504), session=None):
+    session = session or requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+# Scraper function using requests
+def scrape_drugs_com_reviews_requests(drug_name, max_pages=3, delay=2):
     base_url = f"https://www.drugs.com/comments/{drug_name}/"
     all_reviews = []
 
-    async with async_playwright() as p:
-        browser = await p.chromium.launch(headless=False)
-        context = await browser.new_context(
-            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121 Safari/537.36",
-            locale="en-US",
-            viewport={'width': 1280, 'height': 800},
-            device_scale_factor=1,
-            is_mobile=False,
-            has_touch=False
-        )
-        page = await context.new_page()
-
-        for page_num in range(1, max_pages + 1):
-            url = base_url if page_num == 1 else f"{base_url}?page={page_num}"
-            print(f"Scraping: {url}")
-            await page.goto(url, timeout=60000)
-            await asyncio.sleep(delay)  # Give page some time to load
-
-            html = await page.content()
-            await asyncio.sleep(delay)  # Give page some time to load
-            soup = BeautifulSoup(html, 'html.parser')
-            review_blocks = soup.find_all('div', class_='ddc-comment ddc-box ddc-mgb-2')
+    session = requests_retry_session()
+
+    for page_num in range(1, max_pages + 1):
+        url = base_url if page_num == 1 else f"{base_url}?page={page_num}"
+        headers = {"User-Agent": random.choice(USER_AGENTS)}
+
+        try:
+            response = session.get(url, headers=headers, timeout=10)
+            response.raise_for_status()
+
+            soup = BeautifulSoup(response.text, "html.parser")
+            review_blocks = soup.find_all("div", class_="ddc-comment ddc-box ddc-mgb-2")
 
             if not review_blocks:
-                print("No reviews found on this page.")
+                print(f"No reviews found on page {page_num}.")
                 break
 
             for block in review_blocks:
-                review_paragraph = block.find('p')
+                review_paragraph = block.find("p")
+                review_text = None
                 if review_paragraph:
-                    # Remove the <b> tag from the paragraph to isolate the review text
                     if review_paragraph.b:
-                        review_paragraph.b.extract()  # Removes <b> so it doesn't show up in the text
-                        # Get the cleaned text
-                        review_text = review_paragraph.get_text(strip=True)
-
+                        review_paragraph.b.extract()  # remove category (e.g., "For Back Pain")
+                    review_text = review_paragraph.get_text(strip=True)
 
                 all_reviews.append({
-                    "source": url,
-                    "review": review_text if review_text else None
-                    
+                    "review": review_text,
+                    "source": url
                 })
 
-            await asyncio.sleep(delay)
+            time.sleep(delay)  # Polite delay
 
-        await browser.close()
-    return pd.DataFrame(all_reviews)
+        except Exception as e:
+            print(f"Error scraping {url}: {e}")
+            continue
 
+    return pd.DataFrame(all_reviews)
 
 
 final_answer = FinalAnswerTool()