Update scraper.py

scraper.py

@@ -1,128 +1,123 @@
 import time
 import requests
 from bs4 import BeautifulSoup
+from urllib.parse import quote_plus
 
-import requests
-from bs4 import BeautifulSoup
-
-def fetch_soup(url):
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                      "AppleWebKit/537.36 (KHTML, like Gecko) "
-                      "Chrome/114.0.0.0 Safari/537.36",
-        "Accept-Language": "en-US,en;q=0.9",
-    }
-    resp = requests.get(url, headers=headers)
-    resp.raise_for_status()
-    return BeautifulSoup(resp.text, "html.parser")
-    
-    session = requests.Session()
+# Global session with headers
+session = requests.Session()
 session.headers.update({
-    "User-Agent": "Mozilla/5.0 ...",
-    "Accept-Language": "en-US,en;q=0.9",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                  "AppleWebKit/537.36 (KHTML, like Gecko) "
+                  "Chrome/114.0.0.0 Safari/537.36",
+    "Accept-Language": "en-US,en;q=0.9"
 })
-resp = session.get(url)
-
 
 def fetch_soup(url):
-    resp = requests.get(url, headers=HEADERS, timeout=10)
-    resp.raise_for_status()
-    # optional sleep to be polite / avoid rate‑limits
-    time.sleep(1)
-    return BeautifulSoup(resp.text, "html.parser")
+    try:
+        resp = session.get(url, timeout=10)
+        resp.raise_for_status()
+        time.sleep(1)  # polite delay
+        return BeautifulSoup(resp.text, "html.parser")
+    except requests.exceptions.HTTPError as e:
+        print(f"[ERROR] HTTP error while fetching: {url}\n{e}")
+        return BeautifulSoup("", "html.parser")
+    except Exception as e:
+        print(f"[ERROR] General error while fetching: {url}\n{e}")
+        return BeautifulSoup("", "html.parser")
 
 
 def scrape_amazon(query):
     print(f"Scraping Amazon for query: {query}")
-    url = f"https://www.amazon.in/s?k={query}"
+    url = f"https://www.amazon.in/s?k={quote_plus(query)}"
     soup = fetch_soup(url)
 
     products = []
     for item in soup.select("div[data-component-type='s-search-result']"):
-        name_el  = item.select_one("h2 span ")
+        name_el = item.select_one("h2 span")
         price_el = item.select_one("span.a-price-whole")
-        img_el   = item.select_one("img.s-image")
-        link_el  = item.select_one("a")
+        img_el = item.select_one("img.s-image")
+        link_el = item.select_one("a")
 
         products.append({
-            "name":  name_el .get_text(strip=True) if name_el  else "N/A",
+            "name": name_el.get_text(strip=True) if name_el else "N/A",
             "price": price_el.get_text(strip=True) if price_el else "N/A",
-            "img":   img_el  ["src"]           if img_el   else "",
-            "link":  "https://www.amazon.in" + link_el["href"] if link_el else ""
+            "img": img_el["src"] if img_el else "",
+            "link": "https://www.amazon.in" + link_el["href"] if link_el else ""
         })
     return products
 
 
 def scrape_1mg(query):
     print(f"Scraping 1mg for query: {query}")
-    url  = f"https://www.1mg.com/search/all?name={query}"
+    url = f"https://www.1mg.com/search/all?name={quote_plus(query)}"
     soup = fetch_soup(url)
 
     products = []
     for product in soup.select("div.style__horizontal-card___1Zwmt"):
-        name_el  = product.select_one("div.style__product-description___1vPQe")
+        name_el = product.select_one("div.style__product-description___1vPQe")
         price_el = product.select_one("div.style__price-tag___B2csA")
-        link_el  = product.select_one("a")
-        img_el   = product.select_one("img")
+        link_el = product.select_one("a")
+        img_el = product.select_one("img")
 
         products.append({
-            "name":  name_el.get_text(strip=True) if name_el  else "N/A",
+            "name": name_el.get_text(strip=True) if name_el else "N/A",
             "price": price_el.get_text(strip=True) if price_el else "N/A",
-            "img":   img_el  ["src"]           if img_el   else "",
-            "link":  "https://www.1mg.com" + link_el["href"] if link_el else ""
+            "img": img_el["src"] if img_el else "",
+            "link": "https://www.1mg.com" + link_el["href"] if link_el else ""
         })
     return products
-    
+
+
 def scrape_netmeds(query):
     print(f"Scraping Netmeds for query: {query}")
-    url  = f"https://www.netmeds.com/catalogsearch/result/{query}/all"
+    url = f"https://www.netmeds.com/catalogsearch/result/{quote_plus(query)}/all"
     soup = fetch_soup(url)
 
     products = []
     for product in soup.select("div.cat-item"):
-        name_el  = product.select_one("h3")
+        name_el = product.select_one("h3")
         price_el = product.select_one("span.final-price")
-        link_el  = product.select_one("a")
-        img_el   = product.select_one("img")
+        link_el = product.select_one("a")
+        img_el = product.select_one("img")
 
         products.append({
-            "name":  name_el.get_text(strip=True) if name_el  else "N/A",
+            "name": name_el.get_text(strip=True) if name_el else "N/A",
             "price": price_el.get_text(strip=True) if price_el else "N/A",
-            "img":   img_el  ["src"]           if img_el   else "",
-            "link":  "https://www.netmeds.com" + link_el["href"] if link_el else ""
+            "img": img_el["src"] if img_el else "",
+            "link": "https://www.netmeds.com" + link_el["href"] if link_el else ""
         })
     return products
 
+
 def scrape_pharmeasy(query):
     print(f"Scraping PharmEasy for query: {query}")
-    url  = f"https://pharmeasy.in/search/all?name={query}"
+    url = f"https://pharmeasy.in/search/all?name={quote_plus(query)}"
     soup = fetch_soup(url)
 
     products = []
     for product in soup.select("div.ProductCard_medicineUnitContainer__m2_zO"):
-        name_el  = product.select_one("h1")
+        name_el = product.select_one("h1")
         price_el = product.select_one("div.ProductCard_ourPrice__yU5GB")
-        link_el  = product.select_one("a")
-        img_el   = product.select_one("img")
+        link_el = product.select_one("a")
+        img_el = product.select_one("img")
 
         products.append({
-            "name":  name_el.get_text(strip=True) if name_el  else "N/A",
+            "name": name_el.get_text(strip=True) if name_el else "N/A",
             "price": price_el.get_text(strip=True) if price_el else "N/A",
-            "img":   img_el  ["src"]           if img_el   else "",
-            "link":  "https://pharmeasy.in" + link_el["href"] if link_el else ""
+            "img": img_el["src"] if img_el else "",
+            "link": "https://pharmeasy.in" + link_el["href"] if link_el else ""
         })
     return products
 
 
 def search_product(query):
-    amazon_products  = [{"source": "Amazon", **p}   for p in scrape_amazon(query)]
-    one_mg_products  = [{"source": "1mg",   **p}   for p in scrape_1mg(query)]
-    netmeds_products = [{"source": "Netmeds",**p}  for p in scrape_netmeds(query)]
-    pharmeasy_products = [{"source": "PharmEasy",**p} for p in scrape_pharmeasy(query)]
+    amazon_products = [{"source": "Amazon", **p} for p in scrape_amazon(query)]
+    one_mg_products = [{"source": "1mg", **p} for p in scrape_1mg(query)]
+    netmeds_products = [{"source": "Netmeds", **p} for p in scrape_netmeds(query)]
+    pharmeasy_products = [{"source": "PharmEasy", **p} for p in scrape_pharmeasy(query)]
 
     combined = []
-    max_len = max(len(amazon_products), len(one_mg_products),
-                  len(netmeds_products), len(pharmeasy_products))
+    max_len = max(len(amazon_products), len(one_mg_products), len(netmeds_products), len(pharmeasy_products))
 
     for i in range(max_len):
         combined.append({
@@ -130,15 +125,13 @@ def search_product(query):
                 amazon_products[i]["name"] if i < len(amazon_products) else
                 one_mg_products[i]["name"] if i < len(one_mg_products) else
                 netmeds_products[i]["name"] if i < len(netmeds_products) else
-                pharmeasy_products[i]["name"] if i < len(pharmeasy_products) else
-                ""
+                pharmeasy_products[i]["name"] if i < len(pharmeasy_products) else ""
             ),
             "img": (
                 amazon_products[i]["img"] if i < len(amazon_products) else
                 one_mg_products[i]["img"] if i < len(one_mg_products) else
                 netmeds_products[i]["img"] if i < len(netmeds_products) else
-                pharmeasy_products[i]["img"] if i < len(pharmeasy_products) else
-                ""
+                pharmeasy_products[i]["img"] if i < len(pharmeasy_products) else ""
             ),
             "amazon": amazon_products[i] if i < len(amazon_products) else {},
             "one_mg": one_mg_products[i] if i < len(one_mg_products) else {},
@@ -150,44 +143,11 @@ def search_product(query):
 
 
 def search_product_with_progress(query):
-    """Search products with progress tracking - this is for demo purposes"""
-    import requests
-    
-    # Simulate progress tracking by calling each scraper individually
-    amazon_products = [{"source": "Amazon", **p} for p in scrape_amazon(query)]
-    one_mg_products = [{"source": "1mg", **p} for p in scrape_1mg(query)]
-    netmeds_products = [{"source": "Netmeds", **p} for p in scrape_netmeds(query)]
-    pharmeasy_products = [{"source": "PharmEasy", **p} for p in scrape_pharmeasy(query)]
-
-    # Combine results
-    combined = []
-    max_len = max(len(amazon_products), len(one_mg_products),
-                  len(netmeds_products), len(pharmeasy_products))
+    combined = search_product(query)
 
-    for i in range(max_len):
-        combined.append({
-            "name": (
-                amazon_products[i]["name"] if i < len(amazon_products) else
-                one_mg_products[i]["name"] if i < len(one_mg_products) else
-                netmeds_products[i]["name"] if i < len(netmeds_products) else
-                pharmeasy_products[i]["name"] if i < len(pharmeasy_products) else
-                ""
-            ),
-            "img": (
-                amazon_products[i]["img"] if i < len(amazon_products) else
-                one_mg_products[i]["img"] if i < len(one_mg_products) else
-                netmeds_products[i]["img"] if i < len(netmeds_products) else
-                pharmeasy_products[i]["img"] if i < len(pharmeasy_products) else
-                ""
-            ),
-            "amazon": amazon_products[i] if i < len(amazon_products) else {},
-            "one_mg": one_mg_products[i] if i < len(one_mg_products) else {},
-            "netmeds": netmeds_products[i] if i < len(netmeds_products) else {},
-            "pharmeasy": pharmeasy_products[i] if i < len(pharmeasy_products) else {},
-        })
-
-    # Fetch side effects
+    # Fetch side effects using OpenFDA API
     api_url = f'https://api.fda.gov/drug/event.json?api_key=ah8n0nI6468qpS6cMfqvBM0sdGs4nO1sT3Ie7LjJ&search=patient.drug.medicinalproduct:"{query}"'
+
     try:
         response = requests.get(api_url)
         data = response.json()
@@ -199,7 +159,7 @@ def search_product_with_progress(query):
         else:
             side_effects = ["No side effects found."]
     except Exception as e:
-        print(f"Error fetching data: {e}")
-        side_effects = ["Error fetching data. Please try again."]
+        print(f"Error fetching side effects: {e}")
+        side_effects = ["Error fetching side effects."]
 
-    return combined, side_effects
+    return combined, side_effects