Update scraper/utils/HebScraper.py

scraper/utils/HebScraper.py

@@ -207,7 +207,13 @@ class HebScraper:
     
     def search_category_pages(self, category):
         url = f"https://www.heb.com:443/search/?q={category}"
-        response = self.session.get(url)
+        try:
+            response = self.session.get(url)
+        except:
+            try:
+                response = self.session.get(url)
+            except:
+                return 0
         soup = BeautifulSoup(response.text, 'html.parser')
         total_pages = soup.find_all('a', {'data-qe-id': 'paginationListNum'})[-1].text
         print(total_pages)
@@ -229,7 +235,13 @@ class HebScraper:
                 url = f"{self.base_url}{product['product']['productPageURL']}"      
                 urls.append(url)
             print(urls)
-            self.get_all_products_from_category_page(urls)
+            try:
+                self.get_all_products_from_category_page(urls)
+            except:
+                try:        
+                    self.get_all_products_from_category_page(urls)
+                except:
+                    print('Error')
             
     def get_all_products_from_category_page(self, urls):
         for url in urls: