task: async scraper with delay

scraper.py

@@ -33,6 +33,8 @@ async def take_screenshot(url, directory):
         # Desktop screenshot (1920px width)
         page = await browser.new_page(viewport={'width': 1920, 'height': 1080})
         await page.goto(url)
+        # Wait for fade transitions
+        await page.wait_for_timeout(1500)
         # Get full height
         height = await page.evaluate('document.body.scrollHeight')
         await page.set_viewport_size({'width': 1920, 'height': int(height)})
@@ -41,6 +43,8 @@ async def take_screenshot(url, directory):
         # Mobile screenshot (480px width)
         page = await browser.new_page(viewport={'width': 480, 'height': 1080})
         await page.goto(url)
+        # Wait for fade transitions
+        await page.wait_for_timeout(1500)
         # Get full height
         height = await page.evaluate('document.body.scrollHeight')
         await page.set_viewport_size({'width': 480, 'height': int(height)})
@@ -59,14 +63,14 @@ async def scrape_design(design_id):
     
     # Get design page
     response = requests.get(design_url)
-    print(f"Response status: {response.status_code}")
+    print(f"{design_id}: Response status: {response.status_code}")
     
     soup = BeautifulSoup(response.text, "html.parser")
     author_meta = soup.select_one('meta[name="author"]')
     
     # Debug found elements
-    print("\nFound elements:")
-    print(f"h1: {soup.select_one('h1')['content']}")
+    print(f"{design_id}: \nFound elements:")
+    print(f"h1: {soup.select_one('h1').text}")
     print(f"author: {author_meta['content']}")
     
     # Extract metadata with error handling

test_scraper.ipynb

@@ -27,32 +27,39 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "Testing scraper with design 220...\n",
+      "Response status: 200\n",
+      "\n",
+      "Found elements:\n",
+      "h1: CSS Zen Garden\n",
+      "author: Dave Shea\n",
+      "Success!\n",
       "Testing scraper with design 221...\n",
+      "Response status: 200\n",
+      "\n",
+      "Found elements:\n",
+      "h1: CSS Zen Garden\n",
+      "author: Dave Shea\n",
       "Success!\n"
      ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/var/folders/02/z250w46j5_514v22h_ct_zq40000gn/T/ipykernel_37704/2179274543.py:8: RuntimeWarning: coroutine 'scrape_design' was never awaited\n",
-      "  scrape_design(test_design_id)\n",
-      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
-     ]
     }
    ],
    "source": [
     "from scraper import scrape_design\n",
+    "import asyncio\n",
+    "\n",
+    "test_design_ids = [\"220\", \"221\"]\n",
     "\n",
-    "# Test with a single design first\n",
-    "test_design_id = \"221\"\n",
+    "async def test_scraper(ids):\n",
+    "    for test_design_id in ids:\n",
+    "        try:\n",
+    "            print(f\"Testing scraper with design {test_design_id}...\")\n",
+    "            await scrape_design(test_design_id)\n",
+    "            print(\"Success!\")\n",
+    "        except Exception as e:\n",
+    "            print(f\"Error: {str(e)}\")\n",
     "\n",
-    "try:\n",
-    "    print(f\"Testing scraper with design {test_design_id}...\")\n",
-    "    await scrape_design(test_design_id)\n",
-    "    print(\"Success!\")\n",
-    "except Exception as e:\n",
-    "    print(f\"Error: {str(e)}\")"
+    "asyncio.run(test_scraper(test_design_ids))"
    ]
   },
   {