| from src.crawler import Crawler | |
| import logging | |
| # Configure logging to see what's happening | |
| logging.basicConfig(level=logging.INFO) | |
| c = Crawler() | |
| domain = "https://eminentcoders.com/" | |
| print(f"DEBUG: Starting crawl for {domain}") | |
| site_data, discovered, reason = c.crawl_domain(domain, max_pages=1) | |
| print(f"DEBUG: Result - Pages Scanned: {len(site_data)}") | |
| print(f"DEBUG: Discovered: {discovered}") | |
| print(f"DEBUG: Blocked Reason: {reason}") | |