task: adds scraper wait time to load background images

data_collection/notebook.ipynb

@@ -13,72 +13,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Testing scraper with design 001...\n",
-      "001: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 002...\n",
-      "002: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 003...\n",
-      "003: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 004...\n",
-      "004: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 005...\n",
-      "005: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 006...\n",
-      "006: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 007...\n",
-      "007: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 008...\n",
-      "008: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 009...\n",
-      "009: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 010...\n",
-      "010: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 011...\n",
-      "011: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 012...\n",
-      "012: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 013...\n",
-      "013: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 014...\n",
-      "014: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 015...\n",
-      "015: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 016...\n",
-      "016: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 017...\n",
-      "017: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 018...\n",
-      "018: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 019...\n",
-      "019: Response status: 200\n",
-      "Success!\n",
-      "Testing scraper with design 020...\n",
-      "020: Response status: 200\n",
+      "Testing scraper with design 112...\n",
+      "112: Response status: 200\n",
       "Success!\n"
      ]
     }
@@ -88,8 +31,8 @@
     "import asyncio\n",
     "\n",
     "# Pick from a range of 001 to 221\n",
-    "test_set = [f\"{i:03d}\" for i in range(1, 21)]\n",
-    "#test_set = [\"010\"]\n",
+    "#test_set = [f\"{i:03d}\" for i in range(1, 21)]\n",
+    "test_set = [\"112\"]\n",
     "async def test_scraper(ids):\n",
     "    for test_design_id in ids:\n",
     "        try:\n",

data_collection/scraper.py

@@ -31,27 +31,25 @@ async def take_screenshot(url, directory):
         browser = await p.chromium.launch()
         
         # Desktop screenshot (1920px width)
-        page = await browser.new_page(viewport={'width': 1920, 'height': 1080})
+        page = await browser.new_page(viewport={'width': 1600, 'height': 1080})
         await page.goto(url)
         # Wait for network to be idle (no requests for at least 500ms)
-        await page.wait_for_load_state()
-        
-        # Additional wait to ensure any animations/transitions complete
-        #await page.wait_for_timeout(2000)  # 2 second delay
+        await page.wait_for_load_state("networkidle")
+        # Add a significant delay to ensure background images are loaded
+        await page.wait_for_timeout(2000)
         
         # Get full height
         height = await page.evaluate('document.body.scrollHeight')
-        await page.set_viewport_size({'width': 1920, 'height': int(height)})
+        await page.set_viewport_size({'width': 1600, 'height': int(height)})
         await page.screenshot(path=f"{directory}/screenshot_desktop.png", full_page=True)
         
         # Mobile screenshot (480px width)
         page = await browser.new_page(viewport={'width': 480, 'height': 1080})
         await page.goto(url)
         # Wait for network to be idle (no requests for at least 500ms)
-        await page.wait_for_load_state()
-        
-        # Additional wait to ensure any animations/transitions complete
-        #await page.wait_for_timeout(2000)  # 2 second delay
+        await page.wait_for_load_state("networkidle")
+        # Add a significant delay to ensure background images are loaded
+        await page.wait_for_timeout(2000)
         
         # Get full height
         height = await page.evaluate('document.body.scrollHeight')