Upload extractor.py

agents/extractor.py

@@ -443,6 +443,8 @@ class TokenExtractor:
         
         This catches colors defined in CSS rules that may not be
         currently applied to visible elements.
+        
+        Also fetches external stylesheets that may be CORS-blocked.
         """
         css_colors = await page.evaluate("""
             () => {
@@ -493,6 +495,146 @@ class TokenExtractor:
         
         return css_colors
     
+    async def _fetch_external_css_colors(self, page: Page) -> list[dict]:
+        """
+        Fetch and parse external CSS files directly to bypass CORS.
+        
+        This catches colors in external stylesheets that are blocked by CORS.
+        """
+        colors = []
+        
+        try:
+            # Get all stylesheet URLs
+            css_urls = await page.evaluate("""
+                () => {
+                    const urls = [];
+                    const links = document.querySelectorAll('link[rel="stylesheet"]');
+                    links.forEach(link => {
+                        if (link.href) {
+                            urls.push(link.href);
+                        }
+                    });
+                    return urls;
+                }
+            """)
+            
+            # Color regex pattern
+            color_regex = re.compile(r'#[0-9a-fA-F]{3,8}|rgb\([^)]+\)|rgba\([^)]+\)|hsl\([^)]+\)|hsla\([^)]+\)', re.IGNORECASE)
+            
+            # Fetch each CSS file
+            for css_url in css_urls[:10]:  # Limit to 10 files
+                try:
+                    response = await page.request.get(css_url, timeout=5000)
+                    if response.ok:
+                        css_text = await response.text()
+                        
+                        # Find all color values in CSS text
+                        matches = color_regex.findall(css_text)
+                        for match in matches:
+                            colors.append({
+                                "value": match,
+                                "property": "external-css",
+                                "element": "css-file",
+                                "context": "external-stylesheet",
+                            })
+                except Exception as e:
+                    # Skip if fetch fails
+                    pass
+            
+        except Exception as e:
+            self.warnings.append(f"External CSS fetch failed: {str(e)}")
+        
+        return colors
+    
+    async def _extract_all_page_colors(self, page: Page) -> list[dict]:
+        """
+        Extract ALL color values from the page source and styles.
+        
+        This is a brute-force approach that scans the entire page HTML
+        and all style blocks for any color values.
+        """
+        colors = await page.evaluate("""
+            () => {
+                const colors = [];
+                const colorRegex = /#[0-9a-fA-F]{3,8}|rgb\\([^)]+\\)|rgba\\([^)]+\\)|hsl\\([^)]+\\)|hsla\\([^)]+\\)/gi;
+                
+                // 1. Scan all <style> tags
+                const styleTags = document.querySelectorAll('style');
+                styleTags.forEach(style => {
+                    const matches = style.textContent.match(colorRegex);
+                    if (matches) {
+                        matches.forEach(color => {
+                            colors.push({
+                                value: color,
+                                property: 'style-tag',
+                                element: 'style',
+                                context: 'style-block',
+                            });
+                        });
+                    }
+                });
+                
+                // 2. Scan data attributes that might contain colors
+                const allElements = document.querySelectorAll('*');
+                allElements.forEach(el => {
+                    // Check data attributes
+                    for (const attr of el.attributes) {
+                        if (attr.name.startsWith('data-') || attr.name === 'style') {
+                            const matches = attr.value.match(colorRegex);
+                            if (matches) {
+                                matches.forEach(color => {
+                                    colors.push({
+                                        value: color,
+                                        property: attr.name,
+                                        element: el.tagName.toLowerCase(),
+                                        context: 'attribute',
+                                    });
+                                });
+                            }
+                        }
+                    }
+                    
+                    // Check for color in class names (some frameworks use color classes)
+                    const classList = el.className;
+                    if (typeof classList === 'string') {
+                        const colorMatches = classList.match(colorRegex);
+                        if (colorMatches) {
+                            colorMatches.forEach(color => {
+                                colors.push({
+                                    value: color,
+                                    property: 'class',
+                                    element: el.tagName.toLowerCase(),
+                                    context: 'class-name',
+                                });
+                            });
+                        }
+                    }
+                });
+                
+                // 3. Look for colors in script tags (config objects)
+                const scriptTags = document.querySelectorAll('script');
+                scriptTags.forEach(script => {
+                    if (script.textContent && !script.src) {
+                        const matches = script.textContent.match(colorRegex);
+                        if (matches) {
+                            matches.forEach(color => {
+                                colors.push({
+                                    value: color,
+                                    property: 'script',
+                                    element: 'script',
+                                    context: 'javascript',
+                                });
+                            });
+                        }
+                    }
+                });
+                
+                return colors;
+            }
+        """)
+        
+        return colors
+    
     def _process_css_variables(self, css_vars: dict):
         """Process CSS variables and extract color tokens from them."""
         computed = css_vars.get("computed", {})
@@ -530,7 +672,7 @@ class TokenExtractor:
                         contrast_black=round(contrast_black, 2),
                         wcag_aa_large_text=compliance["aa_large_text"],
                         wcag_aa_small_text=compliance["aa_normal_text"],
-                        source=TokenSource.CSS_VARIABLE,
+                        source=TokenSource.DETECTED,  # CSS variable is still "detected"
                         confidence=Confidence.HIGH,
                     )
                 elif hex_value and hex_value in self.colors:
@@ -879,6 +1021,24 @@ class TokenExtractor:
                     except Exception as e:
                         self.warnings.append(f"Stylesheet color extraction failed: {str(e)}")
                     
+                    # 6. Fetch external CSS files (bypass CORS)
+                    external_css_count = 0
+                    try:
+                        external_colors = await self._fetch_external_css_colors(page)
+                        external_css_count = len(external_colors)
+                        self._aggregate_colors(external_colors)
+                    except Exception as e:
+                        self.warnings.append(f"External CSS fetch failed: {str(e)}")
+                    
+                    # 7. Brute-force scan all page content for colors
+                    page_scan_count = 0
+                    try:
+                        page_colors = await self._extract_all_page_colors(page)
+                        page_scan_count = len(page_colors)
+                        self._aggregate_colors(page_colors)
+                    except Exception as e:
+                        self.warnings.append(f"Page scan failed: {str(e)}")
+                    
                     # =========================================================
                     # Log extraction results for this page
                     # =========================================================
@@ -896,6 +1056,8 @@ class TokenExtractor:
                         "svg_colors": svg_color_count,
                         "inline_colors": inline_color_count,
                         "stylesheet_colors": stylesheet_color_count,
+                        "external_css_colors": external_css_count,
+                        "page_scan_colors": page_scan_count,
                         "new_colors": colors_new,
                         "new_typography": typo_new,
                         "new_spacing": spacing_new,