task: ensure design scraper captures images

scraper.py

@@ -33,8 +33,23 @@ async def take_screenshot(url, directory):
         # Desktop screenshot (1920px width)
         page = await browser.new_page(viewport={'width': 1920, 'height': 1080})
         await page.goto(url)
-        # Wait for fade transitions
-        await page.wait_for_timeout(1500)
+        # Wait for network to be idle (no requests for at least 500ms)
+        await page.wait_for_load_state('networkidle')
+        
+        # Wait for all images to be loaded
+        await page.evaluate("""() => {
+            return Promise.all(
+                Array.from(document.images)
+                    .filter(img => !img.complete)
+                    .map(img => new Promise(resolve => {
+                        img.onload = img.onerror = resolve;
+                    }))
+            );
+        }""")
+        
+        # Additional wait to ensure any animations/transitions complete
+        #await page.wait_for_timeout(2000)  # 2 second delay
+        
         # Get full height
         height = await page.evaluate('document.body.scrollHeight')
         await page.set_viewport_size({'width': 1920, 'height': int(height)})
@@ -43,8 +58,23 @@ async def take_screenshot(url, directory):
         # Mobile screenshot (480px width)
         page = await browser.new_page(viewport={'width': 480, 'height': 1080})
         await page.goto(url)
-        # Wait for fade transitions
-        await page.wait_for_timeout(1500)
+        # Wait for network to be idle (no requests for at least 500ms)
+        await page.wait_for_load_state('networkidle')
+        
+        # Wait for all images to be loaded
+        await page.evaluate("""() => {
+            return Promise.all(
+                Array.from(document.images)
+                    .filter(img => !img.complete)
+                    .map(img => new Promise(resolve => {
+                        img.onload = img.onerror = resolve;
+                    }))
+            );
+        }""")
+        
+        # Additional wait to ensure any animations/transitions complete
+        #await page.wait_for_timeout(2000)  # 2 second delay
+        
         # Get full height
         height = await page.evaluate('document.body.scrollHeight')
         await page.set_viewport_size({'width': 480, 'height': int(height)})
@@ -66,18 +96,11 @@ async def scrape_design(design_id):
     print(f"{design_id}: Response status: {response.status_code}")
     
     soup = BeautifulSoup(response.text, "html.parser")
-    author_meta = soup.select_one('meta[name="author"]')
-    
-    # Debug found elements
-    print(f"{design_id}: \nFound elements:")
-    print(f"h1: {soup.select_one('h1').text}")
-    print(f"author: {author_meta['content']}")
     
     # Extract metadata with error handling
     try:
         metadata = {
             "id": design_id,
-            "author": author_meta["content"] if author_meta else "Unknown Author",
             "url": design_url,
             "css_url": css_url
         }