task: csszengarden scraper [poc]

.gitignore

@@ -12,3 +12,6 @@
 
 .chainlit/cache
 
+/designs
+
+__pycache__

pyproject.toml

@@ -12,8 +12,6 @@ dependencies = [
     "ipykernel>=6.29.0",
     "beautifulsoup4>=4.12.0",
     "scrapy>=2.11.0",
-    "selenium>=4.18.0",
-    "selenium-wire>=5.1.0",  # Added for scraper
     "playwright>=1.42.0",
     "pandas>=2.2.0",
     "numpy>=1.26.0",

scraper.py

@@ -2,8 +2,8 @@ import os
 import requests
 from bs4 import BeautifulSoup
 import json
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
+from playwright.async_api import async_playwright
+import asyncio
 
 def create_design_directory(design_id):
     """Create a directory for the design if it doesn't exist"""
@@ -25,32 +25,30 @@ def save_metadata(metadata, directory):
     with open(metadata_path, "w", encoding="utf-8") as f:
         json.dump(metadata, f, indent=4)
 
-def take_screenshot(url, directory):
+async def take_screenshot(url, directory):
     """Take screenshots of the design at desktop and mobile widths"""
-    chrome_options = Options()
-    chrome_options.add_argument("--headless")
-    
-    driver = webdriver.Chrome(options=chrome_options)
-    
-    # Desktop screenshot (1920px width)
-    driver.set_window_size(1920, 1080)
-    driver.get(url)
-    # Wait for page to load and get full height
-    total_height = driver.execute_script("return document.body.scrollHeight")
-    driver.set_window_size(1920, total_height)
-    driver.save_screenshot(f"{directory}/screenshot_desktop.png")
-    
-    # Mobile screenshot (480px width)
-    driver.set_window_size(480, 1080)
-    driver.get(url)
-    # Wait for page to load and get full height
-    total_height = driver.execute_script("return document.body.scrollHeight")
-    driver.set_window_size(480, total_height)
-    driver.save_screenshot(f"{directory}/screenshot_mobile.png")
-    
-    driver.quit()
+    async with async_playwright() as p:
+        browser = await p.chromium.launch()
+        
+        # Desktop screenshot (1920px width)
+        page = await browser.new_page(viewport={'width': 1920, 'height': 1080})
+        await page.goto(url)
+        # Get full height
+        height = await page.evaluate('document.body.scrollHeight')
+        await page.set_viewport_size({'width': 1920, 'height': int(height)})
+        await page.screenshot(path=f"{directory}/screenshot_desktop.png", full_page=True)
+        
+        # Mobile screenshot (480px width)
+        page = await browser.new_page(viewport={'width': 480, 'height': 1080})
+        await page.goto(url)
+        # Get full height
+        height = await page.evaluate('document.body.scrollHeight')
+        await page.set_viewport_size({'width': 480, 'height': int(height)})
+        await page.screenshot(path=f"{directory}/screenshot_mobile.png", full_page=True)
+        
+        await browser.close()
 
-def scrape_design(design_id):
+async def scrape_design(design_id):
     """Scrape a single design"""
     # Create base URLs
     design_url = f"https://www.csszengarden.com/{design_id}"
@@ -63,22 +61,19 @@ def scrape_design(design_id):
     response = requests.get(design_url)
     print(f"Response status: {response.status_code}")
     
-    # Debug HTML content
-    print("\nFirst 500 characters of response:")
-    print(response.text[:500])
-    
     soup = BeautifulSoup(response.text, "html.parser")
+    author_meta = soup.select_one('meta[name="author"]')
     
     # Debug found elements
     print("\nFound elements:")
-    print(f"h1: {soup.select_one('h1')}")
-    print(f"author: {soup.select_one('meta[name=\"author\"]')}")
+    print(f"h1: {soup.select_one('h1')['content']}")
+    print(f"author: {author_meta['content']}")
     
     # Extract metadata with error handling
     try:
         metadata = {
             "id": design_id,
-            "author": soup.select_one('meta[name="author"]')["content"] if soup.select_one('meta[name="author"]') else "Unknown Author",
+            "author": author_meta["content"] if author_meta else "Unknown Author",
             "url": design_url,
             "css_url": css_url
         }
@@ -89,9 +84,9 @@ def scrape_design(design_id):
     # Save everything
     save_css(css_url, directory)
     save_metadata(metadata, directory)
-    take_screenshot(design_url, directory)
+    await take_screenshot(design_url, directory)
 
-def main():
+async def main():
     """Main function to scrape multiple designs"""
     # Create designs directory if it doesn't exist
     if not os.path.exists("designs"):
@@ -103,10 +98,10 @@ def main():
     for design_id in design_ids:
         try:
             print(f"Scraping design {design_id}...")
-            scrape_design(design_id)
+            await scrape_design(design_id)
             print(f"Successfully scraped design {design_id}")
         except Exception as e:
             print(f"Error scraping design {design_id}: {str(e)}")
 
 if __name__ == "__main__":
-    main() 
+    asyncio.run(main()) 

test_scraper.ipynb

@@ -11,7 +11,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!playwright install chromium"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -19,39 +28,28 @@
      "output_type": "stream",
      "text": [
       "Testing scraper with design 221...\n",
-      "Response status: 200\n",
-      "\n",
-      "First 500 characters of response:\n",
-      "<!DOCTYPE html>\n",
-      "<html lang=\"en\">\n",
-      "<head>\n",
-      "\t<meta charset=\"utf-8\">\n",
-      "\t<title>CSS Zen Garden: The Beauty of CSS Design</title>\n",
-      "\n",
-      "\t<link rel=\"stylesheet\" media=\"screen\" href=\"/221/221.css?v=8may2013\">\n",
-      "\t<link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS\" href=\"http://www.csszengarden.com/zengarden.xml\">\n",
-      "\n",
-      "\t<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n",
-      "\t<meta name=\"author\" content=\"Dave Shea\">\n",
-      "\t<meta name=\"description\" content=\"A demonstration of what can be accomplished v\n",
-      "\n",
-      "Found elements:\n",
-      "h1: <h1>CSS Zen Garden</h1>\n",
-      "author: None\n",
-      "author link: None\n",
       "Success!\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/02/z250w46j5_514v22h_ct_zq40000gn/T/ipykernel_37704/2179274543.py:8: RuntimeWarning: coroutine 'scrape_design' was never awaited\n",
+      "  scrape_design(test_design_id)\n",
+      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
+     ]
     }
    ],
    "source": [
-    "from scraper import create_design_directory, save_css, save_metadata, take_screenshot, scrape_design\n",
+    "from scraper import scrape_design\n",
     "\n",
     "# Test with a single design first\n",
     "test_design_id = \"221\"\n",
     "\n",
     "try:\n",
     "    print(f\"Testing scraper with design {test_design_id}...\")\n",
-    "    scrape_design(test_design_id)\n",
+    "    await scrape_design(test_design_id)\n",
     "    print(\"Success!\")\n",
     "except Exception as e:\n",
     "    print(f\"Error: {str(e)}\")"
@@ -59,9 +57,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Files created:\n",
+      "['metadata.json', 'style.css']\n",
+      "\n",
+      "Metadata:\n",
+      "{\n",
+      "  \"id\": \"221\",\n",
+      "  \"author\": \"Dave Shea\",\n",
+      "  \"url\": \"https://www.csszengarden.com/221\",\n",
+      "  \"css_url\": \"https://www.csszengarden.com/221/221.css\"\n",
+      "}\n"
+     ]
+    }
+   ],
    "source": [
     "# If successful, let's check what we got\n",
     "import json\n",

uv.lock

@@ -732,7 +732,7 @@ wheels = [
 [[package]]
 name = "imagineui"
 version = "0.1.0"
-source = { editable = "." }
+source = { virtual = "." }
 dependencies = [
     { name = "beautifulsoup4" },
     { name = "chainlit" },
@@ -748,7 +748,6 @@ dependencies = [
     { name = "python-dotenv" },
     { name = "requests" },
     { name = "scrapy" },
-    { name = "selenium" },
     { name = "sentence-transformers" },
     { name = "torch" },
     { name = "transformers" },
@@ -770,7 +769,6 @@ requires-dist = [
     { name = "python-dotenv", specifier = ">=1.0.0" },
     { name = "requests", specifier = ">=2.28.1" },
     { name = "scrapy", specifier = ">=2.11.0" },
-    { name = "selenium", specifier = ">=4.18.0" },
     { name = "sentence-transformers", specifier = ">=2.5.0" },
     { name = "torch", specifier = "==2.1.2" },
     { name = "transformers", specifier = ">=4.37.0" },
@@ -1861,18 +1859,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/00/f8/bb60a4644287a544ec81df1699d5b965776bc9848d9029d9f9b3402ac8bb/orjson-3.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:cf45e0214c593660339ef63e875f32ddd5aa3b4adc15e662cdb80dc49e194f8e", size = 133570 },
 ]
 
-[[package]]
-name = "outcome"
-version = "1.3.0.post0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "attrs" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692 },
-]
-
 [[package]]
 name = "overrides"
 version = "7.7.0"
@@ -2248,15 +2234,6 @@ version = "2.1.2"
 source = { registry = "https://pypi.org/simple" }
 sdist = { url = "https://files.pythonhosted.org/packages/d5/7b/65f55513d3c769fd677f90032d8d8703e3dc17e88a41b6074d2177548bca/PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2", size = 23224 }
 
-[[package]]
-name = "pysocks"
-version = "1.7.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725 },
-]
-
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -2606,23 +2583,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e8/43/2cc828e9b7a453d791afbe3ef36c951f4641fc1d886b6d39e9455c5468e0/Scrapy-2.12.0-py2.py3-none-any.whl", hash = "sha256:c33e2dc7da42e727390bacb32dd9938a54ac210fa71972b5c392754f478669cd", size = 311170 },
 ]
 
-[[package]]
-name = "selenium"
-version = "4.29.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "certifi" },
-    { name = "trio" },
-    { name = "trio-websocket" },
-    { name = "typing-extensions" },
-    { name = "urllib3", extra = ["socks"] },
-    { name = "websocket-client" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/09de87ef66a10a7d40417d4e93449eb892154d2dc6385187aa9298a2c09d/selenium-4.29.0.tar.gz", hash = "sha256:3a62f7ec33e669364a6c0562a701deb69745b569c50d55f1a912bf8eb33358ba", size = 985717 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2f/a6/fc66ea71ec0769f72abdf15cb9ec9269517abe68a160839383ddff7478f1/selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e", size = 9536642 },
-]
-
 [[package]]
 name = "send2trash"
 version = "1.8.3"
@@ -2704,15 +2664,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 },
 ]
 
-[[package]]
-name = "sortedcontainers"
-version = "2.4.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 },
-]
-
 [[package]]
 name = "soupsieve"
 version = "2.6"
@@ -2982,37 +2933,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/20/37/1f29af63e9c30156a3ed6ebc2754077016577c094f31de7b2631e5d379eb/transformers-4.49.0-py3-none-any.whl", hash = "sha256:6b4fded1c5fee04d384b1014495b4235a2b53c87503d7d592423c06128cbbe03", size = 9970275 },
 ]
 
-[[package]]
-name = "trio"
-version = "0.29.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "attrs" },
-    { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" },
-    { name = "idna" },
-    { name = "outcome" },
-    { name = "sniffio" },
-    { name = "sortedcontainers" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/a1/47/f62e62a1a6f37909aed0bf8f5d5411e06fa03846cfcb64540cd1180ccc9f/trio-0.29.0.tar.gz", hash = "sha256:ea0d3967159fc130acb6939a0be0e558e364fee26b5deeecc893a6b08c361bdf", size = 588952 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c9/55/c4d9bea8b3d7937901958f65124123512419ab0eb73695e5f382521abbfb/trio-0.29.0-py3-none-any.whl", hash = "sha256:d8c463f1a9cc776ff63e331aba44c125f423a5a13c684307e828d930e625ba66", size = 492920 },
-]
-
-[[package]]
-name = "trio-websocket"
-version = "0.12.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "outcome" },
-    { name = "trio" },
-    { name = "wsproto" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8d/ba/ab932f5f520565caf948ccadade04f82daa33272b9629b7bc71fd1bb1a63/trio_websocket-0.12.1.tar.gz", hash = "sha256:d55ccd4d3eae27c494f3fdae14823317839bdcb8214d1173eacc4d42c69fc91b", size = 33547 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d3/b9/b07ec357ba125ad26e1c07781b9d7f0414af85ad76e0d73617ddb5ce041c/trio_websocket-0.12.1-py3-none-any.whl", hash = "sha256:608ec746bb287e5d5a66baf483e41194193c5cf05ffaad6240e7d1fcd80d1e6f", size = 21216 },
-]
-
 [[package]]
 name = "triton"
 version = "2.1.0"
@@ -3115,11 +3035,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 },
 ]
 
-[package.optional-dependencies]
-socks = [
-    { name = "pysocks" },
-]
-
 [[package]]
 name = "uvicorn"
 version = "0.34.0"