Update scraper/utils/HebScraper.py

scraper/utils/HebScraper.py

@@ -226,10 +226,14 @@ class HebScraper:
             self.query['variables']['params']['query'] = category
             # burp0_url = f"https://www.heb.com:443/search/?q={category}&pageNumber={page}"
             url = "https://www.heb.com:443/graphql"
-            response = self.session.post(url, json=self.query)
-            with open('heb.json', 'w+', encoding='utf-8') as file:
-                file.write(response.text)
-            products = response.json()['data']['productSearchV2']['records']  
+            try:
+                response = self.session.post(url, json=self.query)
+                products = response.json()['data']['productSearchV2']['records']  
+            except:
+                self.generate_session()
+                response = self.session.post(url, json=self.query)
+                products = response.json()['data']['productSearchV2']['records']  
+                
             urls = []
             for product in products:
                 url = f"{self.base_url}{product['product']['productPageURL']}"      
@@ -275,6 +279,7 @@ class HebScraper:
         for category in self.categories:
             pages = self.search_category_pages(category)
             self.get_urls_of_category_from_page(category, pages)
+            self.generate_session()
             print(f'Finished {category}')
         return True
 if __name__ == "__main__":