prince1604
Enhance API stability: Add KeepAlive, increase timeout, and optimize crawler threads
f2e524e | import argparse | |
| import sys | |
| from src.crawler import Crawler | |
| from src.analyzer import ImageAnalyzer | |
| from src.reporter import generate_json_report, generate_html_report | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Web Image Alt Text Checker (Whole Domain)") | |
| parser.add_argument("url", help="The Start URL of the domain to crawl (e.g. https://example.com)") | |
| parser.add_argument("--output", choices=['json', 'html', 'both'], default='html', help="Report output format") | |
| parser.add_argument("--limit", type=int, default=100, help="Max pages to crawl (default 100)") | |
| args = parser.parse_args() | |
| # Crawl | |
| print(f"Starting domain crawl from: {args.url}") | |
| crawler = Crawler() | |
| site_data, total_discovered, blocked_reason = crawler.crawl_domain(args.url, max_pages=args.limit) | |
| if not site_data and blocked_reason: | |
| print(f"\n[!] CRAWL BLOCKED: {blocked_reason}") | |
| # Proceed to generate report even if empty to show the block status | |
| elif not site_data: | |
| print("No data crawled. Please check the URL and try again.") | |
| sys.exit(1) | |
| print(f"Crawl complete. Scanned {len(site_data)} pages.") | |
| # Analyze | |
| print("Analyzing images for missing alt text...") | |
| analyzer = ImageAnalyzer() | |
| results = analyzer.analyze_site(site_data) | |
| # Add discovery stats to summary | |
| results['summary']['total_pages_discovered'] = total_discovered | |
| results['summary']['blocked_reason'] = blocked_reason | |
| results['summary']['crawl_blocked'] = bool(blocked_reason) | |
| # Report | |
| print("Generating reports...") | |
| if args.output in ['json', 'both']: | |
| generate_json_report(results, "seo_report.json") | |
| if args.output in ['html', 'both']: | |
| generate_html_report(results, "seo_report.html") | |
| summary = results['summary'] | |
| print("\n--- Summary ---") | |
| if summary.get('crawl_blocked'): | |
| print(f"STATUS: BLOCKED ({summary['blocked_reason']})") | |
| print("Results are likely incomplete.") | |
| print(f"Total Pages Discovered: {summary['total_pages_discovered']}") | |
| print(f"Pages Scanned: {summary['total_pages_scanned']}") | |
| print(f"Total Images: {summary['total_images_found']}") | |
| print(f"Images Missing Alt: {summary['total_images_missing_alt']}") | |
| if summary['total_images_missing_alt'] > 0: | |
| print("Please check the generated report for details.") | |
| elif not summary.get('crawl_blocked'): | |
| print("All images have alt text! Great job.") | |
| if __name__ == "__main__": | |
| main() | |