Upload 2 files

agents/crawler.py

@@ -0,0 +1,366 @@
+"""
+Agent 1: Website Crawler
+Design System Extractor v2
+
+Persona: Meticulous Design Archaeologist
+
+Responsibilities:
+- Auto-discover pages from base URL
+- Classify page types (homepage, listing, detail, etc.)
+- Prepare page list for user confirmation
+"""
+
+import asyncio
+import re
+from urllib.parse import urljoin, urlparse
+from typing import Optional, Callable
+from datetime import datetime
+
+from playwright.async_api import async_playwright, Browser, Page, BrowserContext
+
+from core.token_schema import DiscoveredPage, PageType, Viewport
+from config.settings import get_settings
+
+
+class PageDiscoverer:
+    """
+    Discovers pages from a website for design system extraction.
+    
+    This is the first part of Agent 1's job — finding pages before
+    the human confirms which ones to crawl.
+    """
+    
+    def __init__(self):
+        self.settings = get_settings()
+        self.browser: Optional[Browser] = None
+        self.context: Optional[BrowserContext] = None
+        self.visited_urls: set[str] = set()
+        self.discovered_pages: list[DiscoveredPage] = []
+        
+    async def __aenter__(self):
+        """Async context manager entry."""
+        await self._init_browser()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self._close_browser()
+    
+    async def _init_browser(self):
+        """Initialize Playwright browser."""
+        playwright = await async_playwright().start()
+        self.browser = await playwright.chromium.launch(
+            headless=self.settings.browser.headless
+        )
+        self.context = await self.browser.new_context(
+            viewport={
+                "width": self.settings.viewport.desktop_width,
+                "height": self.settings.viewport.desktop_height,
+            },
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+        )
+    
+    async def _close_browser(self):
+        """Close browser and cleanup."""
+        if self.context:
+            await self.context.close()
+        if self.browser:
+            await self.browser.close()
+    
+    def _normalize_url(self, url: str, base_url: str) -> Optional[str]:
+        """Normalize and validate URL."""
+        # Handle relative URLs
+        if not url.startswith(('http://', 'https://')):
+            url = urljoin(base_url, url)
+        
+        parsed = urlparse(url)
+        base_parsed = urlparse(base_url)
+        
+        # Only allow same domain
+        if parsed.netloc != base_parsed.netloc:
+            return None
+        
+        # Remove fragments and normalize
+        normalized = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+        
+        # Remove trailing slash for consistency
+        if normalized.endswith('/') and len(normalized) > len(f"{parsed.scheme}://{parsed.netloc}/"):
+            normalized = normalized.rstrip('/')
+        
+        return normalized
+    
+    def _classify_page_type(self, url: str, title: str = "") -> PageType:
+        """
+        Classify page type based on URL patterns and title.
+        
+        This is a heuristic — not perfect, but good enough for discovery.
+        """
+        url_lower = url.lower()
+        title_lower = title.lower() if title else ""
+        
+        # Check URL patterns
+        patterns = {
+            PageType.HOMEPAGE: [r'/$', r'/home$', r'/index'],
+            PageType.LISTING: [r'/products', r'/catalog', r'/list', r'/category', r'/collection', r'/search'],
+            PageType.DETAIL: [r'/product/', r'/item/', r'/detail/', r'/p/', r'/[a-z-]+/\d+'],
+            PageType.FORM: [r'/contact', r'/form', r'/apply', r'/submit', r'/register'],
+            PageType.AUTH: [r'/login', r'/signin', r'/signup', r'/auth', r'/account'],
+            PageType.CHECKOUT: [r'/cart', r'/checkout', r'/basket', r'/payment'],
+            PageType.MARKETING: [r'/landing', r'/promo', r'/campaign', r'/offer'],
+            PageType.ABOUT: [r'/about', r'/team', r'/company', r'/story'],
+            PageType.CONTACT: [r'/contact', r'/support', r'/help'],
+        }
+        
+        for page_type, url_patterns in patterns.items():
+            for pattern in url_patterns:
+                if re.search(pattern, url_lower):
+                    return page_type
+        
+        # Check title patterns
+        title_patterns = {
+            PageType.HOMEPAGE: ['home', 'welcome'],
+            PageType.LISTING: ['products', 'catalog', 'collection', 'browse'],
+            PageType.DETAIL: ['product', 'item'],
+            PageType.AUTH: ['login', 'sign in', 'sign up', 'register'],
+            PageType.ABOUT: ['about', 'our story', 'team'],
+            PageType.CONTACT: ['contact', 'get in touch', 'support'],
+        }
+        
+        for page_type, keywords in title_patterns.items():
+            for keyword in keywords:
+                if keyword in title_lower:
+                    return page_type
+        
+        return PageType.OTHER
+    
+    async def _extract_links(self, page: Page, base_url: str) -> list[str]:
+        """Extract all internal links from a page."""
+        links = await page.evaluate("""
+            () => {
+                const links = Array.from(document.querySelectorAll('a[href]'));
+                return links.map(a => a.href).filter(href => 
+                    href && 
+                    !href.startsWith('javascript:') && 
+                    !href.startsWith('mailto:') &&
+                    !href.startsWith('tel:') &&
+                    !href.includes('#')
+                );
+            }
+        """)
+        
+        # Normalize and filter
+        valid_links = []
+        for link in links:
+            normalized = self._normalize_url(link, base_url)
+            if normalized and normalized not in self.visited_urls:
+                valid_links.append(normalized)
+        
+        return list(set(valid_links))
+    
+    async def _get_page_title(self, page: Page) -> str:
+        """Get page title."""
+        try:
+            return await page.title()
+        except Exception:
+            return ""
+    
+    async def discover(
+        self,
+        base_url: str,
+        max_pages: int = None,
+        progress_callback: Optional[Callable[[float], None]] = None
+    ) -> list[DiscoveredPage]:
+        """
+        Discover pages from a website.
+        
+        Args:
+            base_url: The starting URL
+            max_pages: Maximum pages to discover (default from settings)
+            progress_callback: Optional callback for progress updates
+        
+        Returns:
+            List of discovered pages
+        """
+        max_pages = max_pages or self.settings.crawl.max_pages
+        
+        async with self:
+            # Start with homepage
+            normalized_base = self._normalize_url(base_url, base_url)
+            if not normalized_base:
+                raise ValueError(f"Invalid base URL: {base_url}")
+            
+            queue = [normalized_base]
+            self.visited_urls = set()
+            self.discovered_pages = []
+            
+            while queue and len(self.discovered_pages) < max_pages:
+                current_url = queue.pop(0)
+                
+                if current_url in self.visited_urls:
+                    continue
+                
+                self.visited_urls.add(current_url)
+                
+                try:
+                    page = await self.context.new_page()
+                    
+                    # Navigate to page with more lenient settings
+                    # Use 'domcontentloaded' instead of 'networkidle' for faster/more reliable loading
+                    try:
+                        await page.goto(
+                            current_url,
+                            wait_until="domcontentloaded",
+                            timeout=60000  # 60 seconds
+                        )
+                        # Wait a bit more for JS to render
+                        await page.wait_for_timeout(2000)
+                    except Exception as nav_error:
+                        # Try with 'load' event as fallback
+                        try:
+                            await page.goto(
+                                current_url,
+                                wait_until="load",
+                                timeout=60000
+                            )
+                            await page.wait_for_timeout(3000)
+                        except Exception:
+                            # Last resort - just try to get whatever loaded
+                            pass
+                    
+                    # Get page info
+                    title = await self._get_page_title(page)
+                    page_type = self._classify_page_type(current_url, title)
+                    depth = len(urlparse(current_url).path.split('/')) - 1
+                    
+                    # Create discovered page
+                    discovered = DiscoveredPage(
+                        url=current_url,
+                        title=title,
+                        page_type=page_type,
+                        depth=depth,
+                        selected=True,
+                    )
+                    self.discovered_pages.append(discovered)
+                    
+                    # Extract links for further crawling
+                    new_links = await self._extract_links(page, base_url)
+                    
+                    # Prioritize certain page types
+                    priority_patterns = ['/product', '/listing', '/category', '/about', '/contact']
+                    priority_links = [l for l in new_links if any(p in l.lower() for p in priority_patterns)]
+                    other_links = [l for l in new_links if l not in priority_links]
+                    
+                    # Add to queue (priority first)
+                    for link in priority_links + other_links:
+                        if link not in self.visited_urls and link not in queue:
+                            queue.append(link)
+                    
+                    await page.close()
+                    
+                    # Progress callback
+                    if progress_callback:
+                        progress = len(self.discovered_pages) / max_pages
+                        progress_callback(min(progress, 1.0))
+                    
+                    # Rate limiting
+                    await asyncio.sleep(self.settings.crawl.crawl_delay_ms / 1000)
+                    
+                except Exception as e:
+                    # Log error but continue
+                    discovered = DiscoveredPage(
+                        url=current_url,
+                        title="",
+                        page_type=PageType.OTHER,
+                        depth=0,
+                        selected=False,
+                        error=str(e),
+                    )
+                    self.discovered_pages.append(discovered)
+            
+            return self.discovered_pages
+    
+    def get_pages_by_type(self) -> dict[PageType, list[DiscoveredPage]]:
+        """Group discovered pages by type."""
+        grouped: dict[PageType, list[DiscoveredPage]] = {}
+        for page in self.discovered_pages:
+            if page.page_type not in grouped:
+                grouped[page.page_type] = []
+            grouped[page.page_type].append(page)
+        return grouped
+    
+    def get_suggested_pages(self, min_pages: int = None) -> list[DiscoveredPage]:
+        """
+        Get suggested pages for extraction.
+        
+        Ensures diversity of page types and prioritizes key templates.
+        """
+        min_pages = min_pages or self.settings.crawl.min_pages
+        
+        # Priority order for page types
+        priority_types = [
+            PageType.HOMEPAGE,
+            PageType.LISTING,
+            PageType.DETAIL,
+            PageType.FORM,
+            PageType.MARKETING,
+            PageType.AUTH,
+            PageType.ABOUT,
+            PageType.CONTACT,
+            PageType.OTHER,
+        ]
+        
+        selected = []
+        grouped = self.get_pages_by_type()
+        
+        # First pass: get at least one of each priority type
+        for page_type in priority_types:
+            if page_type in grouped and grouped[page_type]:
+                # Take the first (usually shallowest) page of this type
+                page = sorted(grouped[page_type], key=lambda p: p.depth)[0]
+                if page not in selected:
+                    selected.append(page)
+        
+        # Second pass: fill up to min_pages with remaining pages
+        remaining = [p for p in self.discovered_pages if p not in selected and not p.error]
+        remaining.sort(key=lambda p: p.depth)
+        
+        while len(selected) < min_pages and remaining:
+            selected.append(remaining.pop(0))
+        
+        # Mark as selected
+        for page in selected:
+            page.selected = True
+        
+        return selected
+
+
+# =============================================================================
+# CONVENIENCE FUNCTIONS
+# =============================================================================
+
+async def discover_pages(base_url: str, max_pages: int = 20) -> list[DiscoveredPage]:
+    """Convenience function to discover pages."""
+    discoverer = PageDiscoverer()
+    return await discoverer.discover(base_url, max_pages)
+
+
+async def quick_discover(base_url: str) -> dict:
+    """Quick discovery returning summary dict."""
+    pages = await discover_pages(base_url)
+    
+    return {
+        "total_found": len(pages),
+        "by_type": {
+            pt.value: len([p for p in pages if p.page_type == pt])
+            for pt in PageType
+        },
+        "pages": [
+            {
+                "url": p.url,
+                "title": p.title,
+                "type": p.page_type.value,
+                "selected": p.selected,
+            }
+            for p in pages
+        ],
+    }

agents/extractor.py

@@ -0,0 +1,622 @@
+"""
+Agent 1: Token Extractor
+Design System Extractor v2
+
+Persona: Meticulous Design Archaeologist
+
+Responsibilities:
+- Crawl pages at specified viewport
+- Extract computed styles from all elements
+- Collect colors, typography, spacing, radius, shadows
+- Track frequency and context for each token
+"""
+
+import asyncio
+import re
+from typing import Optional, Callable
+from datetime import datetime
+from collections import defaultdict
+
+from playwright.async_api import async_playwright, Browser, Page, BrowserContext
+
+from core.token_schema import (
+    Viewport,
+    ExtractedTokens,
+    ColorToken,
+    TypographyToken,
+    SpacingToken,
+    RadiusToken,
+    ShadowToken,
+    FontFamily,
+    TokenSource,
+    Confidence,
+)
+from core.color_utils import (
+    normalize_hex,
+    parse_color,
+    get_contrast_with_white,
+    get_contrast_with_black,
+    check_wcag_compliance,
+)
+from config.settings import get_settings
+
+
+class TokenExtractor:
+    """
+    Extracts design tokens from web pages.
+    
+    This is the second part of Agent 1's job — after pages are confirmed,
+    we crawl and extract all CSS values.
+    """
+    
+    def __init__(self, viewport: Viewport = Viewport.DESKTOP):
+        self.settings = get_settings()
+        self.viewport = viewport
+        self.browser: Optional[Browser] = None
+        self.context: Optional[BrowserContext] = None
+        
+        # Token collection
+        self.colors: dict[str, ColorToken] = {}
+        self.typography: dict[str, TypographyToken] = {}
+        self.spacing: dict[str, SpacingToken] = {}
+        self.radius: dict[str, RadiusToken] = {}
+        self.shadows: dict[str, ShadowToken] = {}
+        
+        # Font tracking
+        self.font_families: dict[str, FontFamily] = {}
+        
+        # Statistics
+        self.total_elements = 0
+        self.errors: list[str] = []
+        self.warnings: list[str] = []
+    
+    async def __aenter__(self):
+        """Async context manager entry."""
+        await self._init_browser()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self._close_browser()
+    
+    async def _init_browser(self):
+        """Initialize Playwright browser."""
+        playwright = await async_playwright().start()
+        self.browser = await playwright.chromium.launch(
+            headless=self.settings.browser.headless
+        )
+        
+        # Set viewport based on extraction mode
+        if self.viewport == Viewport.DESKTOP:
+            width = self.settings.viewport.desktop_width
+            height = self.settings.viewport.desktop_height
+        else:
+            width = self.settings.viewport.mobile_width
+            height = self.settings.viewport.mobile_height
+        
+        self.context = await self.browser.new_context(
+            viewport={"width": width, "height": height},
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+        )
+    
+    async def _close_browser(self):
+        """Close browser and cleanup."""
+        if self.context:
+            await self.context.close()
+        if self.browser:
+            await self.browser.close()
+    
+    async def _scroll_page(self, page: Page):
+        """Scroll page to load lazy content."""
+        await page.evaluate("""
+            async () => {
+                const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+                const height = document.body.scrollHeight;
+                const step = window.innerHeight;
+                
+                for (let y = 0; y < height; y += step) {
+                    window.scrollTo(0, y);
+                    await delay(100);
+                }
+                
+                // Scroll back to top
+                window.scrollTo(0, 0);
+            }
+        """)
+        
+        # Wait for network idle after scrolling
+        await page.wait_for_load_state("networkidle", timeout=self.settings.browser.network_idle_timeout)
+    
+    async def _extract_styles_from_page(self, page: Page) -> dict:
+        """
+        Extract computed styles from all elements on the page.
+        
+        This is the core extraction logic — we get getComputedStyle for every element.
+        """
+        styles_data = await page.evaluate("""
+            () => {
+                const elements = document.querySelectorAll('*');
+                const results = {
+                    colors: [],
+                    typography: [],
+                    spacing: [],
+                    radius: [],
+                    shadows: [],
+                    elements_count: elements.length,
+                };
+                
+                const colorProperties = [
+                    'color', 'background-color', 'border-color',
+                    'border-top-color', 'border-right-color', 
+                    'border-bottom-color', 'border-left-color',
+                    'outline-color', 'text-decoration-color',
+                ];
+                
+                const spacingProperties = [
+                    'margin-top', 'margin-right', 'margin-bottom', 'margin-left',
+                    'padding-top', 'padding-right', 'padding-bottom', 'padding-left',
+                    'gap', 'row-gap', 'column-gap',
+                ];
+                
+                elements.forEach(el => {
+                    const tag = el.tagName.toLowerCase();
+                    const styles = window.getComputedStyle(el);
+                    
+                    // Skip invisible elements
+                    if (styles.display === 'none' || styles.visibility === 'hidden') {
+                        return;
+                    }
+                    
+                    // --- COLORS ---
+                    colorProperties.forEach(prop => {
+                        const value = styles.getPropertyValue(prop);
+                        if (value && value !== 'rgba(0, 0, 0, 0)' && value !== 'transparent') {
+                            results.colors.push({
+                                value: value,
+                                property: prop,
+                                element: tag,
+                                context: prop.includes('background') ? 'background' : 
+                                        prop.includes('border') ? 'border' : 'text',
+                            });
+                        }
+                    });
+                    
+                    // --- TYPOGRAPHY ---
+                    const fontFamily = styles.getPropertyValue('font-family');
+                    const fontSize = styles.getPropertyValue('font-size');
+                    const fontWeight = styles.getPropertyValue('font-weight');
+                    const lineHeight = styles.getPropertyValue('line-height');
+                    const letterSpacing = styles.getPropertyValue('letter-spacing');
+                    
+                    if (fontSize && fontFamily) {
+                        results.typography.push({
+                            fontFamily: fontFamily,
+                            fontSize: fontSize,
+                            fontWeight: fontWeight,
+                            lineHeight: lineHeight,
+                            letterSpacing: letterSpacing,
+                            element: tag,
+                        });
+                    }
+                    
+                    // --- SPACING ---
+                    spacingProperties.forEach(prop => {
+                        const value = styles.getPropertyValue(prop);
+                        if (value && value !== '0px' && value !== 'auto' && value !== 'normal') {
+                            const px = parseFloat(value);
+                            if (!isNaN(px) && px > 0 && px < 500) {
+                                results.spacing.push({
+                                    value: value,
+                                    valuePx: Math.round(px),
+                                    property: prop,
+                                    context: prop.includes('margin') ? 'margin' : 
+                                            prop.includes('padding') ? 'padding' : 'gap',
+                                });
+                            }
+                        }
+                    });
+                    
+                    // --- BORDER RADIUS ---
+                    const radiusProps = [
+                        'border-radius', 'border-top-left-radius', 
+                        'border-top-right-radius', 'border-bottom-left-radius',
+                        'border-bottom-right-radius',
+                    ];
+                    
+                    radiusProps.forEach(prop => {
+                        const value = styles.getPropertyValue(prop);
+                        if (value && value !== '0px') {
+                            results.radius.push({
+                                value: value,
+                                element: tag,
+                            });
+                        }
+                    });
+                    
+                    // --- BOX SHADOW ---
+                    const shadow = styles.getPropertyValue('box-shadow');
+                    if (shadow && shadow !== 'none') {
+                        results.shadows.push({
+                            value: shadow,
+                            element: tag,
+                        });
+                    }
+                });
+                
+                return results;
+            }
+        """)
+        
+        return styles_data
+    
+    def _process_color(self, color_data: dict) -> Optional[str]:
+        """Process and normalize a color value."""
+        value = color_data.get("value", "")
+        
+        # Parse and normalize
+        parsed = parse_color(value)
+        if not parsed:
+            return None
+        
+        return parsed.hex
+    
+    def _aggregate_colors(self, raw_colors: list[dict]):
+        """Aggregate color data from extraction."""
+        for color_data in raw_colors:
+            hex_value = self._process_color(color_data)
+            if not hex_value:
+                continue
+            
+            if hex_value not in self.colors:
+                # Calculate contrast ratios
+                contrast_white = get_contrast_with_white(hex_value)
+                contrast_black = get_contrast_with_black(hex_value)
+                compliance = check_wcag_compliance(hex_value, "#ffffff")
+                
+                self.colors[hex_value] = ColorToken(
+                    value=hex_value,
+                    frequency=0,
+                    contexts=[],
+                    elements=[],
+                    css_properties=[],
+                    contrast_white=round(contrast_white, 2),
+                    contrast_black=round(contrast_black, 2),
+                    wcag_aa_large_text=compliance["aa_large_text"],
+                    wcag_aa_small_text=compliance["aa_normal_text"],
+                )
+            
+            # Update frequency and context
+            token = self.colors[hex_value]
+            token.frequency += 1
+            
+            context = color_data.get("context", "")
+            if context and context not in token.contexts:
+                token.contexts.append(context)
+            
+            element = color_data.get("element", "")
+            if element and element not in token.elements:
+                token.elements.append(element)
+            
+            prop = color_data.get("property", "")
+            if prop and prop not in token.css_properties:
+                token.css_properties.append(prop)
+    
+    def _aggregate_typography(self, raw_typography: list[dict]):
+        """Aggregate typography data from extraction."""
+        for typo_data in raw_typography:
+            # Create unique key
+            font_family = typo_data.get("fontFamily", "")
+            font_size = typo_data.get("fontSize", "")
+            font_weight = typo_data.get("fontWeight", "400")
+            line_height = typo_data.get("lineHeight", "normal")
+            
+            key = f"{font_size}|{font_weight}|{font_family[:50]}"
+            
+            if key not in self.typography:
+                # Parse font size to px
+                font_size_px = None
+                if font_size.endswith("px"):
+                    try:
+                        font_size_px = float(font_size.replace("px", ""))
+                    except ValueError:
+                        pass
+                
+                # Parse line height
+                line_height_computed = None
+                if line_height and line_height != "normal":
+                    if line_height.endswith("px") and font_size_px:
+                        try:
+                            lh_px = float(line_height.replace("px", ""))
+                            line_height_computed = round(lh_px / font_size_px, 2)
+                        except ValueError:
+                            pass
+                    else:
+                        try:
+                            line_height_computed = float(line_height)
+                        except ValueError:
+                            pass
+                
+                self.typography[key] = TypographyToken(
+                    font_family=font_family.split(",")[0].strip().strip('"\''),
+                    font_size=font_size,
+                    font_size_px=font_size_px,
+                    font_weight=int(font_weight) if font_weight.isdigit() else 400,
+                    line_height=line_height,
+                    line_height_computed=line_height_computed,
+                    letter_spacing=typo_data.get("letterSpacing"),
+                    frequency=0,
+                    elements=[],
+                )
+            
+            # Update
+            token = self.typography[key]
+            token.frequency += 1
+            
+            element = typo_data.get("element", "")
+            if element and element not in token.elements:
+                token.elements.append(element)
+            
+            # Track font families
+            primary_font = token.font_family
+            if primary_font not in self.font_families:
+                self.font_families[primary_font] = FontFamily(
+                    name=primary_font,
+                    fallbacks=[f.strip().strip('"\'') for f in font_family.split(",")[1:]],
+                    frequency=0,
+                )
+            self.font_families[primary_font].frequency += 1
+    
+    def _aggregate_spacing(self, raw_spacing: list[dict]):
+        """Aggregate spacing data from extraction."""
+        for space_data in raw_spacing:
+            value = space_data.get("value", "")
+            value_px = space_data.get("valuePx", 0)
+            
+            key = str(value_px)
+            
+            if key not in self.spacing:
+                self.spacing[key] = SpacingToken(
+                    value=f"{value_px}px",
+                    value_px=value_px,
+                    frequency=0,
+                    contexts=[],
+                    properties=[],
+                    fits_base_4=value_px % 4 == 0,
+                    fits_base_8=value_px % 8 == 0,
+                )
+            
+            token = self.spacing[key]
+            token.frequency += 1
+            
+            context = space_data.get("context", "")
+            if context and context not in token.contexts:
+                token.contexts.append(context)
+            
+            prop = space_data.get("property", "")
+            if prop and prop not in token.properties:
+                token.properties.append(prop)
+    
+    def _aggregate_radius(self, raw_radius: list[dict]):
+        """Aggregate border radius data."""
+        for radius_data in raw_radius:
+            value = radius_data.get("value", "")
+            
+            # Normalize to simple format
+            # "8px 8px 8px 8px" -> "8px"
+            parts = value.split()
+            if len(set(parts)) == 1:
+                value = parts[0]
+            
+            if value not in self.radius:
+                value_px = None
+                if value.endswith("px"):
+                    try:
+                        value_px = int(float(value.replace("px", "")))
+                    except ValueError:
+                        pass
+                
+                self.radius[value] = RadiusToken(
+                    value=value,
+                    value_px=value_px,
+                    frequency=0,
+                    elements=[],
+                    fits_base_4=value_px % 4 == 0 if value_px else False,
+                    fits_base_8=value_px % 8 == 0 if value_px else False,
+                )
+            
+            token = self.radius[value]
+            token.frequency += 1
+            
+            element = radius_data.get("element", "")
+            if element and element not in token.elements:
+                token.elements.append(element)
+    
+    def _aggregate_shadows(self, raw_shadows: list[dict]):
+        """Aggregate box shadow data."""
+        for shadow_data in raw_shadows:
+            value = shadow_data.get("value", "")
+            
+            if value not in self.shadows:
+                self.shadows[value] = ShadowToken(
+                    value=value,
+                    frequency=0,
+                    elements=[],
+                )
+            
+            token = self.shadows[value]
+            token.frequency += 1
+            
+            element = shadow_data.get("element", "")
+            if element and element not in token.elements:
+                token.elements.append(element)
+    
+    def _calculate_confidence(self, frequency: int) -> Confidence:
+        """Calculate confidence level based on frequency."""
+        if frequency >= 10:
+            return Confidence.HIGH
+        elif frequency >= 3:
+            return Confidence.MEDIUM
+        return Confidence.LOW
+    
+    def _detect_spacing_base(self) -> Optional[int]:
+        """Detect the base spacing unit (4 or 8)."""
+        fits_4 = sum(1 for s in self.spacing.values() if s.fits_base_4)
+        fits_8 = sum(1 for s in self.spacing.values() if s.fits_base_8)
+        
+        total = len(self.spacing)
+        if total == 0:
+            return None
+        
+        # If 80%+ values fit base 8, use 8
+        if fits_8 / total >= 0.8:
+            return 8
+        # If 80%+ values fit base 4, use 4
+        elif fits_4 / total >= 0.8:
+            return 4
+        
+        return None
+    
+    async def extract(
+        self,
+        pages: list[str],
+        progress_callback: Optional[Callable[[float], None]] = None
+    ) -> ExtractedTokens:
+        """
+        Extract tokens from a list of pages.
+        
+        Args:
+            pages: List of URLs to crawl
+            progress_callback: Optional callback for progress updates
+        
+        Returns:
+            ExtractedTokens with all discovered tokens
+        """
+        start_time = datetime.now()
+        pages_crawled = []
+        
+        async with self:
+            for i, url in enumerate(pages):
+                try:
+                    page = await self.context.new_page()
+                    
+                    # Navigate with fallback strategy
+                    try:
+                        await page.goto(
+                            url,
+                            wait_until="domcontentloaded",
+                            timeout=60000  # 60 seconds
+                        )
+                        # Wait for JS to render
+                        await page.wait_for_timeout(2000)
+                    except Exception as nav_error:
+                        # Fallback to load event
+                        try:
+                            await page.goto(
+                                url,
+                                wait_until="load",
+                                timeout=60000
+                            )
+                            await page.wait_for_timeout(3000)
+                        except Exception:
+                            self.warnings.append(f"Slow load for {url}, extracting partial content")
+                    
+                    # Scroll to load lazy content
+                    await self._scroll_page(page)
+                    
+                    # Extract styles
+                    styles = await self._extract_styles_from_page(page)
+                    
+                    # Aggregate
+                    self._aggregate_colors(styles.get("colors", []))
+                    self._aggregate_typography(styles.get("typography", []))
+                    self._aggregate_spacing(styles.get("spacing", []))
+                    self._aggregate_radius(styles.get("radius", []))
+                    self._aggregate_shadows(styles.get("shadows", []))
+                    
+                    self.total_elements += styles.get("elements_count", 0)
+                    pages_crawled.append(url)
+                    
+                    await page.close()
+                    
+                    # Progress callback
+                    if progress_callback:
+                        progress_callback((i + 1) / len(pages))
+                    
+                    # Rate limiting
+                    await asyncio.sleep(self.settings.crawl.crawl_delay_ms / 1000)
+                    
+                except Exception as e:
+                    self.errors.append(f"Error extracting {url}: {str(e)}")
+        
+        # Calculate confidence for all tokens
+        for token in self.colors.values():
+            token.confidence = self._calculate_confidence(token.frequency)
+        for token in self.typography.values():
+            token.confidence = self._calculate_confidence(token.frequency)
+        for token in self.spacing.values():
+            token.confidence = self._calculate_confidence(token.frequency)
+        
+        # Detect spacing base
+        spacing_base = self._detect_spacing_base()
+        
+        # Mark outliers in spacing
+        if spacing_base:
+            for token in self.spacing.values():
+                if spacing_base == 8 and not token.fits_base_8:
+                    token.is_outlier = True
+                elif spacing_base == 4 and not token.fits_base_4:
+                    token.is_outlier = True
+        
+        # Determine primary font
+        if self.font_families:
+            primary_font = max(self.font_families.values(), key=lambda f: f.frequency)
+            primary_font.usage = "primary"
+        
+        # Build result
+        end_time = datetime.now()
+        duration_ms = int((end_time - start_time).total_seconds() * 1000)
+        
+        return ExtractedTokens(
+            viewport=self.viewport,
+            source_url=pages[0] if pages else "",
+            pages_crawled=pages_crawled,
+            colors=list(self.colors.values()),
+            typography=list(self.typography.values()),
+            spacing=list(self.spacing.values()),
+            radius=list(self.radius.values()),
+            shadows=list(self.shadows.values()),
+            font_families=list(self.font_families.values()),
+            spacing_base=spacing_base,
+            extraction_timestamp=start_time,
+            extraction_duration_ms=duration_ms,
+            total_elements_analyzed=self.total_elements,
+            unique_colors=len(self.colors),
+            unique_font_sizes=len(set(t.font_size for t in self.typography.values())),
+            unique_spacing_values=len(self.spacing),
+            errors=self.errors,
+            warnings=self.warnings,
+        )
+
+
+# =============================================================================
+# CONVENIENCE FUNCTIONS
+# =============================================================================
+
+async def extract_from_pages(
+    pages: list[str],
+    viewport: Viewport = Viewport.DESKTOP
+) -> ExtractedTokens:
+    """Convenience function to extract tokens from pages."""
+    extractor = TokenExtractor(viewport=viewport)
+    return await extractor.extract(pages)
+
+
+async def extract_both_viewports(pages: list[str]) -> tuple[ExtractedTokens, ExtractedTokens]:
+    """Extract tokens from both desktop and mobile viewports."""
+    desktop_extractor = TokenExtractor(viewport=Viewport.DESKTOP)
+    mobile_extractor = TokenExtractor(viewport=Viewport.MOBILE)
+    
+    desktop_result = await desktop_extractor.extract(pages)
+    mobile_result = await mobile_extractor.extract(pages)
+    
+    return desktop_result, mobile_result