Simon/extract urls from domain (#6)

* Add sitemap parsing utilities and example script for URL extraction

* Add script to extract URLs from legitimate domains using sitemaps and save to CSV

* addade more urls to csv

* Add unit tests for sitemap parser and URL extraction functions

examples/extract_urls_from_sitemaps.py

@@ -0,0 +1,212 @@
+"""Extract URLs from legitimate domains using sitemaps and save to CSV."""
+
+import logging
+import sys
+from pathlib import Path
+import pandas as pd
+from datetime import datetime
+import json
+
+# Add src directory to Python path
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root / "src"))
+
+from phising_detection.data.sitemap_parser import get_urls_from_sitemap
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+def get_already_processed_domains(csv_file: Path) -> set:
+    """
+    Get set of domains that have already been processed.
+
+    Args:
+        csv_file: Path to the CSV file with processed results
+
+    Returns:
+        Set of domain names that have been processed
+    """
+    if not csv_file.exists():
+        return set()
+
+    try:
+        df = pd.read_csv(csv_file)
+        if 'domain' in df.columns:
+            return set(df['domain'].unique())
+    except Exception as e:
+        logger.warning(f"Error reading existing CSV: {e}")
+
+    return set()
+
+
+def save_batch_to_csv(batch_data: list, csv_file: Path):
+    """
+    Save a batch of domain data to CSV.
+
+    Args:
+        batch_data: List of dictionaries with keys: domain, urls, time_updated
+        csv_file: Path to CSV file
+    """
+    if not batch_data:
+        return
+
+    df = pd.DataFrame(batch_data)
+
+    # Convert URL lists to JSON strings for CSV storage
+    df['urls'] = df['urls'].apply(json.dumps)
+
+    # Write header only if creating new file
+    header = not csv_file.exists()
+
+    df.to_csv(csv_file, mode='a', header=header, index=False)
+    logger.info(f"Saved batch of {len(batch_data)} domains to {csv_file}")
+
+
+def main():
+    """Extract URLs from legitimate domains and save to CSV incrementally."""
+
+    # Paths
+    data_dir = Path(__file__).parent.parent / "src" / "phising_detection" / "data" / "data_files"
+    domains_file = data_dir / "legitimate-urls.txt"
+    output_csv = data_dir / "legitimate-urls-extracted.csv"
+
+    # Read domains from file
+    logger.info(f"Reading domains from {domains_file}")
+    with open(domains_file, 'r', encoding='utf-8') as f:
+        all_domains = [line.strip() for line in f if line.strip()]
+
+    logger.info(f"Loaded {len(all_domains)} total domains")
+
+    # Check which domains have already been processed
+    processed_domains = get_already_processed_domains(output_csv)
+    logger.info(f"Already processed: {len(processed_domains)} domains")
+
+    # Filter out already processed domains
+    domains_to_process = [d for d in all_domains if d not in processed_domains]
+    logger.info(f"Remaining to process: {len(domains_to_process)} domains")
+
+    if not domains_to_process:
+        logger.info("All domains have already been processed!")
+        return
+
+    # Configuration
+    max_urls_per_domain = 10
+    batch_size = 50  # Save every 50 domains
+    timeout = 10
+    delay_between_domains = 0.5
+
+    # Process domains in batches
+    total_processed = 0
+    total_urls_extracted = 0
+    domains_with_urls = 0
+    batch_data = []
+
+    for i, domain in enumerate(domains_to_process):
+        logger.info(f"Processing {i+1}/{len(domains_to_process)}: {domain}")
+
+        try:
+            # Get current timestamp
+            time_updated = datetime.now().isoformat()
+
+            # Extract URLs from sitemap
+            urls = get_urls_from_sitemap(
+                domain,
+                max_urls=max_urls_per_domain,
+                timeout=timeout
+            )
+
+            logger.info(f"  Found {len(urls)} URLs from {domain}")
+
+            # Add domain to batch data (even if no URLs found)
+            batch_data.append({
+                'domain': domain,
+                'urls': urls,  # Will be converted to JSON in save function
+                'time_updated': time_updated
+            })
+
+            total_urls_extracted += len(urls)
+            if urls:
+                domains_with_urls += 1
+            total_processed += 1
+
+            # Save batch every N domains
+            if (i + 1) % batch_size == 0:
+                save_batch_to_csv(batch_data, output_csv)
+                logger.info(f"Checkpoint: Processed {total_processed} domains, {domains_with_urls} with URLs, {total_urls_extracted} total URLs")
+                batch_data = []
+
+            # Small delay to be polite
+            import time
+            time.sleep(delay_between_domains)
+
+        except KeyboardInterrupt:
+            logger.info("\nInterrupted by user. Saving current batch...")
+            if batch_data:
+                save_batch_to_csv(batch_data, output_csv)
+            logger.info(f"Saved progress. Processed {total_processed} domains so far.")
+            return
+
+        except Exception as e:
+            logger.error(f"Error processing {domain}: {e}")
+            # Still save the domain with empty URL list
+            batch_data.append({
+                'domain': domain,
+                'urls': [],
+                'time_updated': datetime.now().isoformat()
+            })
+            total_processed += 1
+            continue
+
+    # Save any remaining data
+    if batch_data:
+        save_batch_to_csv(batch_data, output_csv)
+
+    # Print final summary
+    logger.info("\n=== Final Summary ===")
+    logger.info(f"Domains processed this run: {total_processed}")
+    logger.info(f"Domains with URLs this run: {domains_with_urls}")
+    logger.info(f"Total URLs extracted this run: {total_urls_extracted}")
+    if total_processed > 0:
+        logger.info(f"Average URLs per domain: {total_urls_extracted / total_processed:.1f}")
+
+    # Show overall statistics from CSV
+    if output_csv.exists():
+        df = pd.read_csv(output_csv)
+
+        # Parse URL lists from JSON
+        df['urls_parsed'] = df['urls'].apply(json.loads)
+        df['url_count'] = df['urls_parsed'].apply(len)
+
+        total_urls = df['url_count'].sum()
+        domains_with_urls_total = (df['url_count'] > 0).sum()
+
+        logger.info(f"\n=== Overall Statistics ===")
+        logger.info(f"Total domains processed: {len(df)}")
+        logger.info(f"Domains with URLs: {domains_with_urls_total}")
+        logger.info(f"Domains without URLs: {len(df) - domains_with_urls_total}")
+        logger.info(f"Total URLs collected: {total_urls}")
+        if domains_with_urls_total > 0:
+            logger.info(f"Average URLs per domain (with URLs): {total_urls / domains_with_urls_total:.1f}")
+
+        logger.info(f"\n=== Sample Data ===")
+        for _, row in df.head(5).iterrows():
+            url_list = json.loads(row['urls'])
+            url_count = len(url_list)
+            logger.info(f"\n{row['domain']} (updated: {row['time_updated']})")
+            if url_count > 0:
+                logger.info(f"  {url_count} URLs:")
+                for url in url_list[:3]:
+                    logger.info(f"    - {url}")
+                if url_count > 3:
+                    logger.info(f"    ... and {url_count - 3} more")
+            else:
+                logger.info(f"  No URLs found (empty sitemap or no sitemap)")
+
+
+if __name__ == "__main__":
+    main()

examples/load_phishing_urls_example.py

@@ -10,7 +10,7 @@ sys.path.insert(0, str(project_root / "src"))
 from phising_detection.data import load_phishing_urls
 
 # Load phishing URLs from the file
-df = load_phishing_urls("../src/phising_detection/data/phishing-links-ACTIVE.txt")
+df = load_phishing_urls("../src/phising_detection/data/data_files/phishing-links-ACTIVE.txt")
 
 # Display basic information
 print(f"Loaded {len(df)} phishing URLs")

src/phising_detection/data/phishing-links-ACTIVE.txt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9be95a739bde7285f30c7477947f0eaa832a0ad94e0ea197fec81702a7e43036
-size 65832907

src/phising_detection/data/sitemap_parser.py

@@ -0,0 +1,221 @@
+"""Sitemap parsing utilities for extracting URLs from domains."""
+
+import requests
+import xml.etree.ElementTree as ET
+from typing import List, Optional, Set
+from urllib.parse import urljoin
+import time
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_urls_from_sitemap(
+    domain: str,
+    max_urls: Optional[int] = None,
+    timeout: int = 10,
+    max_depth: int = 2
+) -> List[str]:
+    """
+    Extract URLs from a domain's sitemap.xml.
+
+    Args:
+        domain: Domain name (e.g., 'google.com' or 'www.google.com')
+        max_urls: Maximum number of URLs to return (None for all)
+        timeout: Request timeout in seconds
+        max_depth: Maximum depth for nested sitemaps (sitemap index files)
+
+    Returns:
+        List of URLs found in the sitemap
+    """
+    urls: Set[str] = set()
+
+    # Try common sitemap locations
+    sitemap_urls = _get_sitemap_urls(domain)
+
+    for sitemap_url in sitemap_urls:
+        try:
+            logger.info(f"Fetching sitemap: {sitemap_url}")
+            response = requests.get(sitemap_url, timeout=timeout, headers={
+                'User-Agent': 'Mozilla/5.0 (compatible; URLCollector/1.0)'
+            })
+
+            if response.status_code == 200:
+                extracted = _parse_sitemap(
+                    response.content,
+                    max_urls - len(urls) if max_urls else None,
+                    timeout,
+                    max_depth
+                )
+                urls.update(extracted)
+                logger.info(f"Found {len(extracted)} URLs from {sitemap_url}")
+
+                if max_urls and len(urls) >= max_urls:
+                    break
+            else:
+                logger.debug(f"Failed to fetch {sitemap_url}: {response.status_code}")
+
+        except requests.RequestException as e:
+            logger.debug(f"Error fetching {sitemap_url}: {e}")
+            continue
+        except Exception as e:
+            logger.warning(f"Unexpected error parsing {sitemap_url}: {e}")
+            continue
+
+    result = list(urls)
+    if max_urls:
+        result = result[:max_urls]
+
+    return result
+
+
+def _get_sitemap_urls(domain: str) -> List[str]:
+    """
+    Generate possible sitemap URLs for a domain.
+
+    Args:
+        domain: Domain name
+
+    Returns:
+        List of potential sitemap URLs to try
+    """
+    # Remove any protocol if present
+    domain = domain.replace('http://', '').replace('https://', '').rstrip('/')
+
+    # Try both with and without www
+    domains_to_try = [domain]
+    if not domain.startswith('www.'):
+        domains_to_try.append(f'www.{domain}')
+    else:
+        domains_to_try.append(domain.replace('www.', '', 1))
+
+    sitemap_urls = []
+    for d in domains_to_try:
+        # Try HTTPS first, then HTTP
+        sitemap_urls.extend([
+            f'https://{d}/sitemap.xml',
+            f'https://{d}/sitemap_index.xml',
+            f'https://{d}/sitemap',
+            f'http://{d}/sitemap.xml',
+        ])
+
+    return sitemap_urls
+
+
+def _parse_sitemap(
+    content: bytes,
+    max_urls: Optional[int] = None,
+    timeout: int = 10,
+    max_depth: int = 2,
+    current_depth: int = 0
+) -> Set[str]:
+    """
+    Parse sitemap XML content and extract URLs.
+
+    Handles both regular sitemaps and sitemap index files.
+
+    Args:
+        content: XML content as bytes
+        max_urls: Maximum URLs to extract
+        timeout: Request timeout for nested sitemaps
+        max_depth: Maximum recursion depth for sitemap indexes
+        current_depth: Current recursion depth
+
+    Returns:
+        Set of URLs found in the sitemap
+    """
+    urls: Set[str] = set()
+
+    try:
+        root = ET.fromstring(content)
+
+        # Define XML namespaces
+        namespaces = {
+            'sm': 'http://www.sitemaps.org/schemas/sitemap/0.9',
+            'image': 'http://www.google.com/schemas/sitemap-image/1.1',
+            'news': 'http://www.google.com/schemas/sitemap-news/0.9'
+        }
+
+        # Check if this is a sitemap index (contains references to other sitemaps)
+        sitemap_refs = root.findall('.//sm:sitemap/sm:loc', namespaces)
+
+        if sitemap_refs and current_depth < max_depth:
+            # This is a sitemap index - fetch referenced sitemaps
+            logger.info(f"Found sitemap index with {len(sitemap_refs)} sitemaps")
+
+            for sitemap_loc in sitemap_refs:
+                if max_urls and len(urls) >= max_urls:
+                    break
+
+                sitemap_url = sitemap_loc.text
+                if sitemap_url:
+                    try:
+                        response = requests.get(sitemap_url, timeout=timeout, headers={
+                            'User-Agent': 'Mozilla/5.0 (compatible; URLCollector/1.0)'
+                        })
+                        if response.status_code == 200:
+                            nested_urls = _parse_sitemap(
+                                response.content,
+                                max_urls - len(urls) if max_urls else None,
+                                timeout,
+                                max_depth,
+                                current_depth + 1
+                            )
+                            urls.update(nested_urls)
+                            time.sleep(0.1)  # Small delay to be polite
+                    except Exception as e:
+                        logger.debug(f"Error fetching nested sitemap {sitemap_url}: {e}")
+                        continue
+
+        # Extract regular URL entries
+        url_entries = root.findall('.//sm:url/sm:loc', namespaces)
+
+        for loc in url_entries:
+            if max_urls and len(urls) >= max_urls:
+                break
+            if loc.text:
+                urls.add(loc.text)
+
+    except ET.ParseError as e:
+        logger.warning(f"XML parse error: {e}")
+    except Exception as e:
+        logger.warning(f"Error parsing sitemap: {e}")
+
+    return urls
+
+
+def extract_urls_from_domains(
+    domains: List[str],
+    max_urls_per_domain: int = 10,
+    timeout: int = 10,
+    delay_between_domains: float = 0.5
+) -> dict:
+    """
+    Extract URLs from multiple domains using their sitemaps.
+
+    Args:
+        domains: List of domain names
+        max_urls_per_domain: Maximum URLs to extract per domain
+        timeout: Request timeout in seconds
+        delay_between_domains: Delay in seconds between domain requests
+
+    Returns:
+        Dictionary mapping domain to list of extracted URLs
+    """
+    results = {}
+
+    for i, domain in enumerate(domains):
+        logger.info(f"Processing domain {i+1}/{len(domains)}: {domain}")
+
+        urls = get_urls_from_sitemap(
+            domain,
+            max_urls=max_urls_per_domain,
+            timeout=timeout
+        )
+
+        results[domain] = urls
+
+        if i < len(domains) - 1:  # Don't sleep after the last domain
+            time.sleep(delay_between_domains)
+
+    return results

tests/test_sitemap_parser.py

@@ -0,0 +1,407 @@
+"""Tests for sitemap parser module."""
+
+import pytest
+import responses
+from phising_detection.data.sitemap_parser import (
+    get_urls_from_sitemap,
+    _get_sitemap_urls,
+    _parse_sitemap,
+    extract_urls_from_domains
+)
+
+
+class TestGetSitemapUrls:
+    """Tests for _get_sitemap_urls function."""
+
+    def test_basic_domain(self):
+        """Test sitemap URL generation for basic domain."""
+        urls = _get_sitemap_urls("example.com")
+
+        assert "https://example.com/sitemap.xml" in urls
+        assert "https://example.com/sitemap_index.xml" in urls
+        assert "https://example.com/sitemap" in urls
+        assert "http://example.com/sitemap.xml" in urls
+
+    def test_domain_with_www(self):
+        """Test sitemap URL generation for domain with www."""
+        urls = _get_sitemap_urls("www.example.com")
+
+        # Should try both with and without www
+        assert any("www.example.com" in url for url in urls)
+        assert any("example.com/sitemap" in url and "www" not in url for url in urls)
+
+    def test_domain_without_www(self):
+        """Test sitemap URL generation adds www variant."""
+        urls = _get_sitemap_urls("example.com")
+
+        assert any("www.example.com" in url for url in urls)
+
+
+class TestParseSitemap:
+    """Tests for _parse_sitemap function."""
+
+    def test_parse_simple_sitemap(self):
+        """Test parsing a simple sitemap with URLs."""
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url>
+        <loc>https://example.com/page1</loc>
+    </url>
+    <url>
+        <loc>https://example.com/page2</loc>
+    </url>
+    <url>
+        <loc>https://example.com/page3</loc>
+    </url>
+</urlset>"""
+
+        urls = _parse_sitemap(sitemap_xml)
+
+        assert len(urls) == 3
+        assert "https://example.com/page1" in urls
+        assert "https://example.com/page2" in urls
+        assert "https://example.com/page3" in urls
+
+    def test_parse_empty_sitemap(self):
+        """Test parsing an empty sitemap."""
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</urlset>"""
+
+        urls = _parse_sitemap(sitemap_xml)
+        assert len(urls) == 0
+
+    def test_parse_sitemap_with_max_urls(self):
+        """Test parsing sitemap with max_urls limit."""
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example.com/page1</loc></url>
+    <url><loc>https://example.com/page2</loc></url>
+    <url><loc>https://example.com/page3</loc></url>
+    <url><loc>https://example.com/page4</loc></url>
+    <url><loc>https://example.com/page5</loc></url>
+</urlset>"""
+
+        urls = _parse_sitemap(sitemap_xml, max_urls=3)
+        assert len(urls) == 3
+
+    def test_parse_invalid_xml(self):
+        """Test parsing invalid XML returns empty set."""
+        invalid_xml = b"This is not XML"
+
+        urls = _parse_sitemap(invalid_xml)
+        assert len(urls) == 0
+
+    @responses.activate
+    def test_parse_sitemap_index(self):
+        """Test parsing a sitemap index that references other sitemaps."""
+        sitemap_index_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <sitemap>
+        <loc>https://example.com/sitemap1.xml</loc>
+    </sitemap>
+    <sitemap>
+        <loc>https://example.com/sitemap2.xml</loc>
+    </sitemap>
+</sitemapindex>"""
+
+        sitemap1_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example.com/page1</loc></url>
+    <url><loc>https://example.com/page2</loc></url>
+</urlset>"""
+
+        sitemap2_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example.com/page3</loc></url>
+</urlset>"""
+
+        # Mock the nested sitemap requests
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap1.xml",
+            body=sitemap1_xml,
+            status=200
+        )
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap2.xml",
+            body=sitemap2_xml,
+            status=200
+        )
+
+        urls = _parse_sitemap(sitemap_index_xml, max_depth=2)
+
+        assert len(urls) == 3
+        assert "https://example.com/page1" in urls
+        assert "https://example.com/page2" in urls
+        assert "https://example.com/page3" in urls
+
+
+class TestGetUrlsFromSitemap:
+    """Tests for get_urls_from_sitemap function."""
+
+    @responses.activate
+    def test_successful_sitemap_fetch(self):
+        """Test successful sitemap fetching."""
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example.com/page1</loc></url>
+    <url><loc>https://example.com/page2</loc></url>
+</urlset>"""
+
+        # Mock the sitemap request
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap.xml",
+            body=sitemap_xml,
+            status=200
+        )
+
+        urls = get_urls_from_sitemap("example.com", timeout=5)
+
+        assert len(urls) >= 2
+        assert "https://example.com/page1" in urls
+        assert "https://example.com/page2" in urls
+
+    @responses.activate
+    def test_sitemap_not_found(self):
+        """Test handling when sitemap is not found."""
+        # Mock 404 responses for all sitemap URLs
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap_index.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "http://example.com/sitemap.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example.com/sitemap.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example.com/sitemap_index.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example.com/sitemap",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "http://www.example.com/sitemap.xml",
+            status=404
+        )
+
+        urls = get_urls_from_sitemap("example.com", timeout=5)
+
+        assert len(urls) == 0
+
+    @responses.activate
+    def test_max_urls_limit(self):
+        """Test that max_urls limit is respected."""
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example.com/page1</loc></url>
+    <url><loc>https://example.com/page2</loc></url>
+    <url><loc>https://example.com/page3</loc></url>
+    <url><loc>https://example.com/page4</loc></url>
+    <url><loc>https://example.com/page5</loc></url>
+</urlset>"""
+
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap.xml",
+            body=sitemap_xml,
+            status=200
+        )
+
+        urls = get_urls_from_sitemap("example.com", max_urls=3, timeout=5)
+
+        assert len(urls) <= 3
+
+    @responses.activate
+    def test_timeout_handling(self):
+        """Test that timeout is handled gracefully."""
+        from requests.exceptions import Timeout
+
+        # Mock timeout
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap.xml",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap_index.xml",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "https://example.com/sitemap",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "http://example.com/sitemap.xml",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example.com/sitemap.xml",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example.com/sitemap_index.xml",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example.com/sitemap",
+            body=Timeout()
+        )
+        responses.add(
+            responses.GET,
+            "http://www.example.com/sitemap.xml",
+            body=Timeout()
+        )
+
+        # Should not raise exception, just return empty list
+        urls = get_urls_from_sitemap("example.com", timeout=1)
+        assert len(urls) == 0
+
+
+class TestExtractUrlsFromDomains:
+    """Tests for extract_urls_from_domains function."""
+
+    @responses.activate
+    def test_extract_from_multiple_domains(self):
+        """Test extracting URLs from multiple domains."""
+        sitemap1_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example1.com/page1</loc></url>
+    <url><loc>https://example1.com/page2</loc></url>
+</urlset>"""
+
+        sitemap2_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example2.com/pageA</loc></url>
+</urlset>"""
+
+        # Mock sitemaps for both domains
+        responses.add(
+            responses.GET,
+            "https://example1.com/sitemap.xml",
+            body=sitemap1_xml,
+            status=200
+        )
+        responses.add(
+            responses.GET,
+            "https://example2.com/sitemap.xml",
+            body=sitemap2_xml,
+            status=200
+        )
+
+        domains = ["example1.com", "example2.com"]
+        results = extract_urls_from_domains(
+            domains,
+            max_urls_per_domain=10,
+            timeout=5,
+            delay_between_domains=0
+        )
+
+        assert "example1.com" in results
+        assert "example2.com" in results
+        assert len(results["example1.com"]) == 2
+        assert len(results["example2.com"]) == 1
+        assert "https://example1.com/page1" in results["example1.com"]
+        assert "https://example2.com/pageA" in results["example2.com"]
+
+    @responses.activate
+    def test_extract_with_failures(self):
+        """Test extraction continues even if some domains fail."""
+        sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    <url><loc>https://example1.com/page1</loc></url>
+</urlset>"""
+
+        # Mock success for first domain
+        responses.add(
+            responses.GET,
+            "https://example1.com/sitemap.xml",
+            body=sitemap_xml,
+            status=200
+        )
+
+        # Mock failures for second domain
+        responses.add(
+            responses.GET,
+            "https://example2.com/sitemap.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://example2.com/sitemap_index.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://example2.com/sitemap",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "http://example2.com/sitemap.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example2.com/sitemap.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example2.com/sitemap_index.xml",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "https://www.example2.com/sitemap",
+            status=404
+        )
+        responses.add(
+            responses.GET,
+            "http://www.example2.com/sitemap.xml",
+            status=404
+        )
+
+        domains = ["example1.com", "example2.com"]
+        results = extract_urls_from_domains(
+            domains,
+            max_urls_per_domain=10,
+            timeout=5,
+            delay_between_domains=0
+        )
+
+        # Should have results for both, but example2 should be empty
+        assert "example1.com" in results
+        assert "example2.com" in results
+        assert len(results["example1.com"]) == 1
+        assert len(results["example2.com"]) == 0