Update app.py

app.py

@@ -5,18 +5,12 @@ import requests
 import traceback
 from flask import Flask, request, jsonify
 from bs4 import BeautifulSoup
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.chrome.service import Service
-from webdriver_manager.chrome import ChromeDriverManager
-import time
-import re
-import base64
 import logging
-from PIL import Image
-from io import BytesIO
 import cssutils
+import re
 import urllib.parse
+from PIL import Image
+from io import BytesIO
 
 app = Flask(__name__)
 
@@ -24,16 +18,6 @@ app = Flask(__name__)
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# Configure Chrome options for headless browsing
-def get_chrome_options():
-    options = Options()
-    options.add_argument("--headless")
-    options.add_argument("--no-sandbox")
-    options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("--disable-gpu")
-    options.add_argument("--window-size=1440,900")
-    return options
-
 @app.route('/')
 def home():
     return '''
@@ -123,272 +107,255 @@ def convert_website():
         
         logger.info(f"Converting website: {url} with viewport width: {viewport_width}")
         
-        # Set up Chrome driver with webdriver_manager
-        options = get_chrome_options()
-        options.add_argument(f"--window-size={viewport_width},{viewport_height}")
-        
-        # Use ChromeDriverManager to handle driver installation
-        service = Service(ChromeDriverManager().install())
-        driver = webdriver.Chrome(service=service, options=options)
-        driver.set_window_size(viewport_width, viewport_height)
-        
         try:
-            # Load the page
-            driver.get(url)
-            
-            # Wait for page to load (adjust as needed)
-            time.sleep(5)
+            # Use requests to get the webpage
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml',
+                'Accept-Language': 'en-US,en;q=0.9',
+            }
             
-            # Get page dimensions
-            page_dimensions = driver.execute_script("""
-                return {
-                    width: Math.max(document.body.scrollWidth, document.documentElement.scrollWidth),
-                    height: Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)
-                };
-            """)
+            response = requests.get(url, headers=headers, timeout=15)
+            response.raise_for_status()  # Raise an exception for HTTP errors
             
-            viewport_height = page_dimensions['height']
+            # Parse the HTML content
+            soup = BeautifulSoup(response.text, 'html.parser')
             
-            # Parse the page elements
-            elements = extract_elements(driver)
+            # Extract the page elements using BeautifulSoup
+            elements = extract_elements_bs(soup)
             
-            # Get all CSS styles
-            styles = extract_styles(driver)
+            # Extract CSS styles
+            styles = extract_styles_bs(soup, url)
             
             # Apply styles to elements
             apply_styles_to_elements(elements, styles)
             
+            # Estimate page height based on content (simplified approach)
+            estimated_height = viewport_height
+            if elements:
+                # Find the maximum y-coordinate plus height
+                max_y = 0
+                for element in elements:
+                    elem_bottom = element.get('y', 0) + element.get('height', 0)
+                    max_y = max(max_y, elem_bottom)
+                estimated_height = max(viewport_height, max_y)
+            
             # Prepare response
             response = {
                 "status": "success",
                 "url": url,
                 "viewport_width": viewport_width,
-                "viewport_height": viewport_height,
+                "viewport_height": estimated_height,
                 "elements": elements
             }
             
             return jsonify(response)
             
-        finally:
-            driver.quit()
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Request error: {str(e)}")
+            return jsonify({"error": f"Failed to fetch website: {str(e)}"}), 500
             
     except Exception as e:
         logger.error(f"Error: {str(e)}")
         logger.error(traceback.format_exc())
         return jsonify({"error": str(e), "traceback": traceback.format_exc()}), 500
 
-def extract_elements(driver):
-    """Extract elements from the webpage"""
+def extract_elements_bs(soup):
+    """Extract elements from the webpage using BeautifulSoup"""
     elements = []
     
-    # Execute JavaScript to extract elements
-    elements_data = driver.execute_script("""
-        function getElementInfo(element, depth = 0) {
-            if (!element || depth > 15) return null;
-            
-            // Skip invisible elements
-            if (element.offsetWidth === 0 || element.offsetHeight === 0) return null;
-            
-            // Get element position and size
-            const rect = element.getBoundingClientRect();
-            if (rect.width < 1 || rect.height < 1) return null;
-            
-            // Create element data
-            const elementData = {
-                tagName: element.tagName.toLowerCase(),
-                id: element.id || null,
-                className: element.className || null,
-                x: rect.left,
-                y: rect.top,
-                width: rect.width,
-                height: rect.height
-            };
-            
-            // Handle text elements
-            if (element.tagName.toLowerCase() === 'p' || 
-                element.tagName.toLowerCase() === 'h1' || 
-                element.tagName.toLowerCase() === 'h2' || 
-                element.tagName.toLowerCase() === 'h3' || 
-                element.tagName.toLowerCase() === 'h4' || 
-                element.tagName.toLowerCase() === 'h5' || 
-                element.tagName.toLowerCase() === 'h6' || 
-                element.tagName.toLowerCase() === 'span' || 
-                element.tagName.toLowerCase() === 'a') {
-                
-                const textContent = element.textContent.trim();
-                if (textContent) {
-                    elementData.type = 'text';
-                    elementData.content = textContent;
-                    
-                    // Get computed style
-                    const style = window.getComputedStyle(element);
-                    elementData.style = {
-                        color: style.color,
-                        fontSize: style.fontSize,
-                        fontWeight: style.fontWeight,
-                        lineHeight: style.lineHeight,
-                        textAlign: style.textAlign,
-                        fontFamily: style.fontFamily
-                    };
-                }
-            }
-            
-            // Handle image elements
-            else if (element.tagName.toLowerCase() === 'img') {
-                elementData.type = 'image';
-                elementData.src = element.src;
-                elementData.alt = element.alt || '';
-            }
-            
-            // Handle div/container elements
-            else if (element.tagName.toLowerCase() === 'div' || 
-                    element.tagName.toLowerCase() === 'section' || 
-                    element.tagName.toLowerCase() === 'article' || 
-                    element.tagName.toLowerCase() === 'header' || 
-                    element.tagName.toLowerCase() === 'footer' || 
-                    element.tagName.toLowerCase() === 'main') {
-                
-                elementData.type = 'div';
-                
-                // Get background color
-                const style = window.getComputedStyle(element);
-                elementData.style = {
-                    backgroundColor: style.backgroundColor,
-                    borderRadius: style.borderRadius,
-                    borderWidth: style.borderWidth,
-                    borderColor: style.borderColor,
-                    boxShadow: style.boxShadow
-                };
-                
-                // Check if there are direct text children
-                if (element.childNodes.length > 0) {
-                    let directText = '';
-                    for (let i = 0; i < element.childNodes.length; i++) {
-                        if (element.childNodes[i].nodeType === Node.TEXT_NODE) {
-                            const text = element.childNodes[i].textContent.trim();
-                            if (text) directText += text + ' ';
-                        }
-                    }
-                    
-                    if (directText.trim()) {
-                        elementData.type = 'text';
-                        elementData.content = directText.trim();
-                        elementData.style.color = style.color;
-                        elementData.style.fontSize = style.fontSize;
-                        elementData.style.fontWeight = style.fontWeight;
-                    }
-                }
-                
-                // Get children
-                const children = [];
-                for (let i = 0; i < element.children.length; i++) {
-                    const childData = getElementInfo(element.children[i], depth + 1);
-                    if (childData) children.push(childData);
-                }
-                
-                if (children.length > 0) {
-                    elementData.children = children;
-                    elementData.type = 'container';
-                }
-            }
-            
-            // Handle button elements
-            else if (element.tagName.toLowerCase() === 'button' || 
-                    (element.tagName.toLowerCase() === 'a' && window.getComputedStyle(element).display === 'inline-block')) {
-                
-                elementData.type = 'rectangle';
-                elementData.name = 'Button';
-                
-                // Get style
-                const style = window.getComputedStyle(element);
-                elementData.style = {
-                    backgroundColor: style.backgroundColor,
-                    color: style.color,
-                    borderRadius: style.borderRadius,
-                    borderWidth: style.borderWidth,
-                    borderColor: style.borderColor
-                };
-                
-                // Add text content
-                const textContent = element.textContent.trim();
-                if (textContent) {
-                    elementData.content = textContent;
-                }
-            }
-            
-            else {
-                // Default to rectangle for other elements
-                elementData.type = 'rectangle';
-                
-                // Get style
-                const style = window.getComputedStyle(element);
-                elementData.style = {
-                    backgroundColor: style.backgroundColor
-                };
-                
-                // Get children
-                const children = [];
-                for (let i = 0; i < element.children.length; i++) {
-                    const childData = getElementInfo(element.children[i], depth + 1);
-                    if (childData) children.push(childData);
-                }
-                
-                if (children.length > 0) {
-                    elementData.children = children;
-                }
-            }
-            
-            return elementData;
+    # Helper function to get the coordinates and dimensions
+    # For simplicity, we'll just stack elements vertically
+    y_position = 0
+    max_width = 1440  # Default max width
+    
+    # Process body and its children
+    body = soup.find('body')
+    if not body:
+        return elements
+    
+    # Add the body as a container
+    body_element = {
+        'type': 'container',
+        'tagName': 'body',
+        'x': 0,
+        'y': 0,
+        'width': max_width,
+        'height': 900,  # Default height
+        'children': []
+    }
+    
+    # Process main content elements
+    for element in body.find_all(['div', 'header', 'main', 'footer', 'section', 'nav'], recursive=False):
+        element_data = process_element(element, 0, y_position, max_width)
+        if element_data:
+            y_position += element_data['height'] + 10  # Add spacing
+            body_element['children'].append(element_data)
+    
+    # Adjust body height
+    body_element['height'] = y_position + 50  # Add some padding
+    
+    # Make the body itself the first element, and return its children
+    elements = body_element['children']
+    
+    return elements
+
+def process_element(element, x_position, y_position, max_width, depth=0):
+    """Process a single HTML element"""
+    if depth > 10:  # Limit recursion depth
+        return None
+    
+    tag_name = element.name.lower()
+    element_height = 50  # Default height for elements
+    
+    # Skip script, style tags
+    if tag_name in ['script', 'style', 'meta', 'link']:
+        return None
+    
+    # Create element data dictionary
+    element_data = {
+        'tagName': tag_name,
+        'x': x_position,
+        'y': y_position,
+        'width': max_width,
+        'height': element_height,
+    }
+    
+    # Get element classes
+    element_classes = element.get('class', [])
+    if element_classes:
+        if isinstance(element_classes, list):
+            element_data['className'] = ' '.join(element_classes)
+        else:
+            element_data['className'] = element_classes
+    
+    # Get element ID
+    element_id = element.get('id')
+    if element_id:
+        element_data['id'] = element_id
+    
+    # Handle different types of elements
+    if tag_name in ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'span', 'a']:
+        element_data['type'] = 'text'
+        text_content = element.get_text().strip()
+        element_data['content'] = text_content
+        element_height = 20  # Default height for text
+        
+        # Adjust height based on text length
+        if text_content:
+            num_lines = len(text_content) // 50 + 1  # Rough estimate
+            element_height = max(20, num_lines * 20)  # Min 20px, 20px per line
+        
+        # Basic style properties
+        element_data['style'] = {
+            'color': '#000000',
+            'fontSize': '16px',
+            'fontWeight': 'normal',
+            'lineHeight': '1.5'
         }
         
-        function getVisibleElements() {
-            const bodyElement = document.body;
-            const result = getElementInfo(bodyElement);
-            return result ? result.children || [] : [];
+        # Adjust style based on tag
+        if tag_name.startswith('h'):
+            size = 26 - int(tag_name[1])  # h1 = 25px, h2 = 24px, etc.
+            element_data['style']['fontSize'] = f"{size}px"
+            element_data['style']['fontWeight'] = 'bold'
+            element_height = size * 1.5  # Adjust height based on font size
+    
+    elif tag_name == 'img':
+        element_data['type'] = 'image'
+        element_data['src'] = element.get('src', '')
+        element_data['alt'] = element.get('alt', '')
+        element_height = 200  # Default height for images
+    
+    elif tag_name in ['div', 'section', 'article', 'header', 'footer', 'main', 'nav']:
+        # Container elements
+        element_data['type'] = 'div'
+        element_data['style'] = {
+            'backgroundColor': 'transparent',
+            'borderRadius': '0px'
         }
         
-        return getVisibleElements();
-    """)
+        # Process children
+        children = []
+        child_y_position = 0
+        
+        for child in element.find_all(['div', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'img', 'span', 'a', 'section', 'article', 'nav'], recursive=False):
+            child_data = process_element(child, 5, child_y_position, max_width - 10, depth + 1)
+            if child_data:
+                children.append(child_data)
+                child_y_position += child_data['height'] + 5  # Add spacing
+        
+        if children:
+            element_data['children'] = children
+            element_data['type'] = 'container'
+            element_height = child_y_position + 10  # Total height of children + padding
+        else:
+            # Check if there's direct text content
+            text_content = element.get_text(strip=True)
+            if text_content:
+                element_data['type'] = 'text'
+                element_data['content'] = text_content
+                element_data['style'] = {
+                    'color': '#000000',
+                    'fontSize': '16px'
+                }
+                element_height = 40  # Default height for text containers
+    
+    # Update height
+    element_data['height'] = element_height
     
-    return elements_data
+    return element_data
 
-def extract_styles(driver):
-    """Extract CSS styles from the webpage"""
+def extract_styles_bs(soup, base_url):
+    """Extract CSS styles from the webpage using BeautifulSoup"""
     styles = {}
     
-    # Execute JavaScript to extract styles
-    css_rules = driver.execute_script("""
-        const sheets = document.styleSheets;
-        const rules = [];
-        
-        for (let i = 0; i < sheets.length; i++) {
-            try {
-                const sheet = sheets[i];
-                const ruleList = sheet.rules || sheet.cssRules;
-                
-                for (let j = 0; j < ruleList.length; j++) {
-                    try {
-                        const rule = ruleList[j];
-                        if (rule.selectorText) {
-                            rules.push({
-                                selector: rule.selectorText,
-                                style: rule.style.cssText
-                            });
-                        }
-                    } catch (e) {
-                        // Skip rule if it can't be accessed
-                    }
-                }
-            } catch (e) {
-                // Skip stylesheet if it can't be accessed
-            }
-        }
-        
-        return rules;
-    """)
+    # Extract inline styles
+    for element in soup.find_all(style=True):
+        classes = element.get('class', [])
+        if classes:
+            class_str = '.'.join(classes)
+            styles[f".{class_str}"] = element['style']
     
-    for rule in css_rules:
-        styles[rule['selector']] = rule['style']
+    # Extract style tags
+    for style_tag in soup.find_all('style'):
+        css_text = style_tag.string
+        if css_text:
+            parsed_styles = parse_css(css_text)
+            styles.update(parsed_styles)
+    
+    # Extract linked stylesheets
+    for link in soup.find_all('link', rel='stylesheet'):
+        href = link.get('href')
+        if href:
+            # Make absolute URL if relative
+            if not href.startswith(('http://', 'https://')):
+                href = urllib.parse.urljoin(base_url, href)
+            
+            try:
+                css_response = requests.get(href, timeout=5)
+                if css_response.ok:
+                    parsed_styles = parse_css(css_response.text)
+                    styles.update(parsed_styles)
+            except Exception as e:
+                logger.warning(f"Failed to fetch stylesheet {href}: {e}")
+    
+    return styles
+
+def parse_css(css_text):
+    """Parse CSS text into a dictionary of selectors and styles"""
+    styles = {}
+    try:
+        sheet = cssutils.parseString(css_text)
+        for rule in sheet:
+            if rule.type == rule.STYLE_RULE:
+                selector = rule.selectorText
+                style_dict = {}
+                for property in rule.style:
+                    style_dict[property.name] = property.value
+                styles[selector] = style_dict
+    except Exception as e:
+        logger.warning(f"CSS parsing error: {e}")
     
     return styles
 
@@ -411,37 +378,24 @@ def apply_styles_to_elements(elements, styles):
         if 'children' in element and element['children']:
             apply_styles_to_elements(element['children'], styles)
 
-def apply_style(element, style_text):
+def apply_style(element, style_dict):
     """Apply a CSS style to an element"""
     if 'style' not in element:
         element['style'] = {}
         
-    # Parse CSS text to extract properties
-    style_dict = parse_css_text(style_text)
-    
     # Apply properties to element style
-    for key, value in style_dict.items():
-        if key not in element['style'] or not element['style'][key]:
+    if isinstance(style_dict, dict):
+        for key, value in style_dict.items():
             element['style'][key] = value
-
-def parse_css_text(css_text):
-    """Parse CSS text into a dictionary"""
-    style_dict = {}
-    
-    # Basic parsing of CSS text
-    for item in css_text.split(';'):
-        if ':' in item:
-            key, value = item.split(':', 1)
-            key = key.strip()
-            value = value.strip()
-            if key and value:
-                # Convert to camelCase for JavaScript
-                key_parts = key.split('-')
-                if len(key_parts) > 1:
-                    key = key_parts[0] + ''.join(part.capitalize() for part in key_parts[1:])
-                style_dict[key] = value
-                
-    return style_dict
+    elif isinstance(style_dict, str):
+        # Parse inline style string
+        for item in style_dict.split(';'):
+            if ':' in item:
+                key, value = item.split(':', 1)
+                key = key.strip()
+                value = value.strip()
+                if key and value:
+                    element['style'][key] = value
 
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", 7860))