Update app.py

app.py

@@ -32,44 +32,56 @@ def scrape_drugs_com_reviews(drug_name, max_pages=3, delay=2):
     """
     Scrapes user reviews from Drugs.com for a given drug.
     """
-    base_url = f"https://www.drugs.com/comments/{drug_name}/"
+        base_url = f"https://www.drugs.com/comments/{drug_name}/"
     all_reviews = []
 
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True)
-        page = browser.new_page()
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=False)
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121 Safari/537.36",
+            locale="en-US",
+            viewport={'width': 1280, 'height': 800},
+            device_scale_factor=1,
+            is_mobile=False,
+            has_touch=False
+        )
+        page = await context.new_page()
 
         for page_num in range(1, max_pages + 1):
             url = base_url if page_num == 1 else f"{base_url}?page={page_num}"
             print(f"Scraping: {url}")
-            page.goto(url, timeout=60000)
-            time.sleep(delay)  # Give page some time to load
+            await page.goto(url, timeout=60000)
+            await asyncio.sleep(delay)  # Give page some time to load
 
-            html = page.content()
+            html = await page.content()
+            await asyncio.sleep(delay)  # Give page some time to load
             soup = BeautifulSoup(html, 'html.parser')
-            review_blocks = soup.select('.user-comment')
+            print(soup)
+            review_blocks = soup.find_all('div', class_='ddc-comment ddc-box ddc-mgb-2')
 
             if not review_blocks:
                 print("No reviews found on this page.")
                 break
 
             for block in review_blocks:
-                review_text = block.select_one('.user-comment-text')
-                condition = block.select_one('.drug-condition')
-                rating = block.select_one('.rating-score')
-                date = block.select_one('.comment-date')
+                review_paragraph = block.find('p')
+                if review_paragraph:
+                    # Remove the <b> tag from the paragraph to isolate the review text
+                    if review_paragraph.b:
+                        review_paragraph.b.extract()  # Removes <b> so it doesn't show up in the text
+                        # Get the cleaned text
+                        review_text = review_paragraph.get_text(strip=True)
+
 
                 all_reviews.append({
-                    "condition": condition.get_text(strip=True) if condition else None,
-                    "rating": rating.get_text(strip=True) if rating else None,
-                    "review": review_text.get_text(strip=True) if review_text else None,
-                    "date": date.get_text(strip=True) if date else None,
+
+                    "review": review_text if review_text else None,
                     "source": url
                 })
 
-            time.sleep(delay)
+            await asyncio.sleep(delay)
 
-        browser.close()
+        await browser.close()
     return pd.DataFrame(all_reviews)