Update scraper/utils/WallmartScraper.py

scraper/utils/WallmartScraper.py

@@ -5,7 +5,7 @@ import csv
 import json
 import time
 from .DatabaseDataSaver import save_product
-
+from undetected_chromedriver import Chrome, ChromeOptions
 class WallmartScraper:
     def __init__(self):
         self.ac = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
@@ -14,6 +14,20 @@ class WallmartScraper:
         self.categories = [
             'health', 'beauty', 'personal care'
         ]
+        self.generate_session()
+        
+    def generate_session(self):
+        options = ChromeOptions()
+        options.add_argument("--headless")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--no-sandbox")
+
+        driver = Chrome(options=options)
+        driver.get("https://www.walmart.com/")
+        print(driver.get_cookies())
+        cookies = driver.get_cookies()
+        [self.session.cookies.set(cookie['name'], cookie['value']) for cookie in cookies]
+
     def get_product_detail(self, url):
         response = self.session.get(
             url, headers=self.headers)
@@ -124,7 +138,3 @@ class WallmartScraper:
         return True
 
 
-if __name__ == "__main__":
-    print('Starting...')
-    scraper = HebScraper()
-    response = scraper.run()