Update scraper.py

scraper.py

@@ -14,85 +14,125 @@ HEADERS = {
 def fetch_soup(url):
     resp = requests.get(url, headers=HEADERS, timeout=10)
     resp.raise_for_status()
-    time.sleep(1)  # Be polite and avoid rate-limiting
+    time.sleep(1)  # Be polite and avoid rate limits
     return BeautifulSoup(resp.text, "html.parser")
 
 def scrape_amazon(query):
+    print(f"Scraping Amazon for query: {query}")
     url = f"https://www.amazon.in/s?k={query}"
-    soup = fetch_soup(url)
-    products = []
-    for item in soup.select("div.s-result-item"):
-        name = item.select_one("h2 a span")
-        price = item.select_one("span.a-price-whole")
-        img = item.select_one("img.s-image")
-        link = item.select_one("h2 a")
-        products.append({
-            "name": name.get_text(strip=True) if name else "N/A",
-            "price": price.get_text(strip=True) if price else "N/A",
-            "img": img["src"] if img else "",
-            "link": "https://www.amazon.in" + link["href"] if link else ""
-        })
-    return products
+    try:
+        soup = fetch_soup(url)
+        products = []
+        for item in soup.select("div.s-result-item"):
+            name = item.select_one("span.a-text-normal")
+            name = name.get_text(strip=True) if name else "N/A"
+            price = item.select_one("span.a-price-whole")
+            price = price.get_text(strip=True) if price else "N/A"
+            img = item.select_one("img.s-image")
+            img = img["src"] if img else ""
+            link_el = item.select_one("a.a-link-normal.s-no-outline")
+            link = "https://www.amazon.in" + link_el["href"] if link_el else ""
+            if name != "N/A":  # Only include if product name exists
+                products.append({
+                    "name": name,
+                    "price": price,
+                    "img": img,
+                    "link": link
+                })
+        return products
+    except requests.exceptions.RequestException as e:
+        print(f"Error scraping Amazon: {e}")
+        return []
 
 def scrape_1mg(query):
-    url = f"https://www.1mg.com/search/all?name={query}"
-    soup = fetch_soup(url)
-    products = []
-    for item in soup.select("div.style__product-box___3KfT3"):
-        name = item.select_one("div.style__pro-title___3G3M6")
-        price = item.select_one("div.style__price-tag___B2csA")
-        img = item.select_one("img.style__product-image___1bCpL")
-        link = item.select_one("a.style__product-link___1hWpa")
-        products.append({
-            "name": name.get_text(strip=True) if name else "N/A",
-            "price": price.get_text(strip=True) if price else "N/A",
-            "img": img["src"] if img else "",
-            "link": "https://www.1mg.com" + link["href"] if link else ""
-        })
-    return products
+    print(f"Scraping 1mg for query: {query}")
+    url = "https://www.1mg.com/api/v5/search/results"
+    payload = {
+        "term": query,
+        "pageNumber": 1,
+        "pageSize": 20,
+        "type": "product"
+    }
+    try:
+        resp = requests.post(url, json=payload, headers=HEADERS)
+        resp.raise_for_status()
+        data = resp.json()
+        products = []
+        for item in data.get("data", {}).get("products", []):
+            name = item.get("name", "N/A")
+            price = item.get("mrp", "N/A")
+            img = item.get("image_url", "")  # Adjust key based on actual response
+            link = item.get("product_url", "")
+            products.append({
+                "name": name,
+                "price": price,
+                "img": img,
+                "link": link
+            })
+        return products
+    except requests.exceptions.RequestException as e:
+        print(f"Error scraping 1mg: {e}")
+        return []
 
 def scrape_netmeds(query):
-    url = f"https://www.netmeds.com/catalogsearch/result/{query}/all"
-    soup = fetch_soup(url)
-    products = []
-    for item in soup.select("div.product-item"):
-        name = item.select_one("a.product-name")
-        price = item.select_one("span.final-price")
-        img = item.select_one("img.product-image")
-        link = item.select_one("a.product-name")
-        products.append({
-            "name": name.get_text(strip=True) if name else "N/A",
-            "price": price.get_text(strip=True) if price else "N/A",
-            "img": img["src"] if img else "",
-            "link": "https://www.netmeds.com" + link["href"] if link else ""
-        })
-    return products
+    print(f"Scraping Netmeds for query: {query}")
+    url = f"https://www.netmeds.com/mservice/rest/v2/product/search?term={query}&page=1&per_page=20"
+    try:
+        resp = requests.get(url, headers=HEADERS)
+        resp.raise_for_status()
+        data = resp.json()
+        products = []
+        for item in data.get("products", []):
+            name = item.get("name", "N/A")
+            price = item.get("price", "N/A")
+            img = item.get("image", "")
+            link = item.get("url", "")
+            products.append({
+                "name": name,
+                "price": price,
+                "img": img,
+                "link": link
+            })
+        return products
+    except requests.exceptions.RequestException as e:
+        print(f"Error scraping Netmeds: {e}")
+        return []
 
 def scrape_pharmeasy(query):
-    url = f"https://pharmeasy.in/search/all?name={query}"
-    soup = fetch_soup(url)
-    products = []
-    for item in soup.select("div.ooufh"):
-        name = item.select_one("a._3v1HN")
-        price = item.select_one("div._1_yM9")
-        img = item.select_one("img._1nYRD")
-        link = item.select_one("a._3v1HN")
-        products.append({
-            "name": name.get_text(strip=True) if name else "N/A",
-            "price": price.get_text(strip=True) if price else "N/A",
-            " personally": img["src"] if img else "",
-            "link": "https://pharmeasy.in" + link["href"] if link else ""
-        })
-    return products
+    print(f"Scraping PharmEasy for query: {query}")
+    url = f"https://pharmeasy.in/api/search/search?q={query}"
+    try:
+        resp = requests.get(url, headers=HEADERS)
+        resp.raise_for_status()
+        data = resp.json()
+        products = []
+        for item in data.get("data", {}).get("products", []):
+            name = item.get("name", "N/A")
+            price = item.get("sale_price", "N/A")
+            img = item.get("image", "")
+            link = item.get("slug", "")
+            if link:
+                link = f"https://pharmeasy.in/product/{link}"
+            products.append({
+                "name": name,
+                "price": price,
+                "img": img,
+                "link": link
+            })
+        return products
+    except requests.exceptions.RequestException as e:
+        print(f"Error scraping PharmEasy: {e}")
+        return []
 
 def search_product(query):
-    amazon_products = scrape_amazon(query)
-    one_mg_products = scrape_1mg(query)
-    netmeds_products = scrape_netmeds(query)
-    pharmeasy_products = scrape_pharmeasy(query)
+    amazon_products = [{"source": "Amazon", **p} for p in scrape_amazon(query)]
+    one_mg_products = [{"source": "1mg", **p} for p in scrape_1mg(query)]
+    netmeds_products = [{"source": "Netmeds", **p} for p in scrape_netmeds(query)]
+    pharmeasy_products = [{"source": "PharmEasy", **p} for p in scrape_pharmeasy(query)]
 
     combined = []
-    max_len = max(len(amazon_products), len(one_mg_products), len(netmeds_products), len(pharmeasy_products))
+    max_len = max(len(amazon_products), len(one_mg_products),
+                  len(netmeds_products), len(pharmeasy_products))
 
     for i in range(max_len):
         combined.append({
@@ -115,10 +155,6 @@ def search_product(query):
             "netmeds": netmeds_products[i] if i < len(netmeds_products) else {},
             "pharmeasy": pharmeasy_products[i] if i < len(pharmeasy_products) else {},
         })
+
     return combined
 
-# Example usage (uncomment to test):
-# if __name__ == "__main__":
-#     products = search_product("paracetamol")
-#     for product in products:
-#         print(product)