Update crawler.py

crawler.py

@@ -14,4 +14,115 @@ from backend.ai_interpreter import pattern_based_interpreter
 logger = logging.getLogger("webtapi.crawler")
 
 # Rest of your crawler.py code remains exactly the same...
-# [Keep all your existing functions and implementation]
+# [Keep all your existing functions and implementation]
+
+
+
+
+def get_random_user_agent():
+    """Return a random user agent to avoid detection"""
+    user_agents = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15"
+    ]
+    return random.choice(user_agents)
+
+def get_domain(url):
+    """Extract domain from URL"""
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc}"
+
+def is_same_domain(url, base_url):
+    """Check if URL belongs to the same domain"""
+    return get_domain(url) == get_domain(base_url)
+
+def get_links(html_content, base_url):
+    """Extract all links from HTML content"""
+    soup = BeautifulSoup(html_content, 'lxml')
+    links = set()
+    
+    for a in soup.find_all('a', href=True):
+        href = a['href']
+        if href.startswith(('#', 'javascript:', 'mailto:', 'tel:')):
+            continue
+        
+        # Resolve relative URLs
+        full_url = urljoin(base_url, href)
+        
+        # Only include links from the same domain
+        if is_same_domain(full_url, base_url):
+            # Normalize URL by removing fragments and query parameters
+            parsed_url = urlparse(full_url)
+            normalized_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
+            if normalized_url:
+                links.add(normalized_url)
+    
+    return list(links)
+
+def fetch_page(url):
+    """Fetch a page with error handling"""
+    try:
+        headers = {
+            "User-Agent": get_random_user_agent(),
+            "Accept-Language": "en-US,en;q=0.9",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Encoding": "gzip, deflate",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1"
+        }
+        
+        response = requests.get(url, headers=headers, timeout=15)
+        response.raise_for_status()
+        return response.text, True
+    except Exception as e:
+        logger.error(f"Failed to fetch {url}: {str(e)}")
+        return None, False
+
+def crawl_website(start_url, extraction_plan, max_pages=20, max_depth=3, delay=1):
+    """
+    Crawl a website using BFS algorithm
+    """
+    domain = get_domain(start_url)
+    visited = set()
+    queue = deque([(start_url, 0)])  # (url, depth)
+    results = []
+    page_count = 0
+    
+    while queue and page_count < max_pages:
+        url, depth = queue.popleft()
+        
+        if url in visited or depth > max_depth:
+            continue
+            
+        visited.add(url)
+        logger.info(f"Crawling: {url} (depth: {depth})")
+        
+        # Fetch the page
+        html_content, success = fetch_page(url)
+        if not success:
+            continue
+            
+        # Extract data from the page (we'll process this later)
+        page_data = {
+            "url": url,
+            "depth": depth,
+            "html_content": html_content,
+            "status": "success"
+        }
+        results.append(page_data)
+        page_count += 1
+        
+        # Get links from this page for further crawling
+        if depth < max_depth:
+            links = get_links(html_content, url)
+            for link in links:
+                if link not in visited and link not in [u for u, d in queue]:
+                    queue.append((link, depth + 1))
+        
+        # Respectful delay
+        time.sleep(delay)
+    
+    return results