Initial commit for deployment

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.10 (webscreper)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (webscreper)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/webscreper.iml" filepath="$PROJECT_DIR$/.idea/webscreper.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

.idea/webscreper.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.10 (webscreper)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

DEPLOYMENT_GUIDE.md

@@ -0,0 +1,102 @@
+# 🚀 Deployment Guide (Free & Lifetime)
+
+This guide will show you how to deploy your Web Scraper API to **Render.com** for free. Render is an excellent cloud platform that supports Python and Playwright out of the box.
+
+## ✅ Prerequisites
+
+1.  **GitHub Account**: You need a GitHub account to host your code.
+2.  **Git Installed**: You must have Git installed on your computer.
+
+---
+
+## Step 1: Push Code to GitHub
+
+First, we need to get your code onto GitHub.
+
+1.  **Log in to GitHub** and create a **New Repository**.
+    *   Name it something like `seo-scraper-api`.
+    *   Make it **Public** or **Private** (Private is fine).
+    *   **Do not** initialize with a README (we have code locally).
+
+2.  **Open your terminal** in the project folder (`d:\webscreper`) and run these commands one by one:
+
+    ```bash
+    # Initialize git
+    git init
+
+    # Add all files
+    git add .
+
+    # Commit changes
+    git commit -m "Initial commit for deployment"
+
+    # Link to your new GitHub repo (Replace YOUR_USERNAME and REPO_NAME)
+    git remote add origin https://github.com/YOUR_USERNAME/seo-scraper-api.git
+
+    # Push code
+    git push -u origin master
+    ```
+
+---
+tney aavdi gyu imean bolta 
+## Step 2: Create a Web Service on Render
+
+1.  Go to [dashboard.render.com](https://dashboard.render.com) and sign up/log in.
+2.  Click the **"New +"** button and select **"Web Service"**.
+3.  Connect your **GitHub** account.
+4.  Find your `seo-scraper-api` repository and click **"Connect"**.
+
+---
+
+## Step 3: Configure the Service
+
+Fill in the details exactly as follows to ensure it works with Flask and Playwright:
+
+*   **Name**: `seo-scraper` (or whatever you like)
+*   **Region**: Closest to you (e.g., Frankfurt or Oregon)
+*   **Branch**: `master`
+*   **Root Directory**: (Leave blank)
+*   **Runtime**: **Python 3**
+*   **Build Command**:
+    ```bash
+    ./build.sh
+    ```
+*   **Start Command**:
+    ```bash
+    gunicorn api:app
+    ```
+*   **Instance Type**: Select **Free**
+
+### ⚠️ crucial: Environment Variables
+
+Scroll down to "Advanced" or "Environment Variables" and make sure these settings are correct. We already set up the `build.sh` script to handle dependencies, so this part should be automatic.
+
+---
+
+## Step 4: Deploy
+
+1.  Click **"Create Web Service"**.
+2.  Render will start building your app. You can watch the logs.
+    *   It will install Python dependencies.
+    *   It will install Chromium (for Playwright).
+3.  Once finished, you will see a green checkmark **"Live"**.
+4.  Copy your URL (e.g., `https://seo-scraper.onrender.com`).
+
+---
+
+## Step 5: Test Your API
+
+Now you can use this URL in Postman instead of `127.0.0.1:5050`.
+
+**New Request:**
+*   **Method**: `GET`
+*   **URL**: `https://YOUR-APP-NAME.onrender.com/api/seo-report?domain=https://example.com`
+
+---
+
+## ℹ️ Important Notes about "Free" Tier
+
+1.  **Spin Down**: The free instance "sleeps" after 15 minutes of inactivity. The first request after sleeping might take 30-50 seconds to wake it up. This is normal.
+2.  **Keep Alive (Optional)**: If you want it to stay awake, you can use a free "Uptime Monitor" (like UptimeRobot) to ping your API every 10 minutes.
+
+You now have a fully functional, free, lifetime deployment of your scraper! 🚀

Procfile

@@ -0,0 +1 @@
+web: gunicorn api:app

api.py

@@ -0,0 +1,94 @@
+from flask import Flask, jsonify, request
+from flask_cors import CORS
+import json
+import os
+import random
+from src.crawler import Crawler
+from src.analyzer import ImageAnalyzer
+
+app = Flask(__name__)
+CORS(app)
+
+REPORT_FILE = 'seo_report.json'
+
+@app.route('/api/seo-report', methods=['GET', 'POST'])
+def get_seo_report():
+
+    def get_param(name, default):
+        val = request.args.get(name) or request.form.get(name)
+        if val is None and request.is_json:
+            val = request.json.get(name)
+        return val if val is not None else default
+
+    domain = get_param('domain', None)
+    limit = int(get_param('limit', 25))
+
+    if domain:
+        try:
+            print(f"Starting live scan for: {domain}")
+            
+            # Initialize Crawler
+            crawler = Crawler()
+            # Crawl the domain with the requested limit
+            site_data, total_discovered, _ = crawler.crawl_domain(domain, max_pages=limit)
+            
+            if not site_data:
+                return jsonify({
+                    "summary": {
+                        "total_pages_scanned": 0,
+                        "total_images_found": 0,
+                        "total_images_missing_alt": 0,
+                        "total_pages_discovered": 0
+                    },
+                    "details": []
+                })
+
+            # Analyze Results
+            analyzer = ImageAnalyzer()
+            results = analyzer.analyze_site(site_data)
+            
+            # Add discovery stats
+            results['summary']['total_pages_discovered'] = total_discovered
+            results['details_count'] = len(results['details'])
+            
+            return jsonify(results)
+
+        except Exception as e:
+            return jsonify({"error": f"Scraping failed: {str(e)}"}), 500
+
+    if not os.path.exists(REPORT_FILE):
+        return jsonify({"error": "Report file not found. Please run result logic first."}), 404
+
+    try:
+        with open(REPORT_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        
+        # Get parameters for filtering existing report
+        # limit is already extracted above
+        random_param = str(get_param('random', 'true')).lower()
+        is_random = random_param == 'true'
+
+        summary = data.get('summary', {})
+        details = data.get('details', [])
+
+        # Filter details if limit is provided
+        if limit is not None and limit > 0:
+            if is_random and limit < len(details):
+                details = random.sample(details, limit)
+            else:
+                details = details[:limit]
+        
+        response = {
+            "summary": summary,
+            "details_count": len(details),  # Useful logic for client
+            "details": details
+        }
+        
+        return jsonify(response)
+
+    except Exception as e:
+        return jsonify({"error": f"Failed to read report: {str(e)}"}), 500
+
+if __name__ == '__main__':
+    # Run on 0.0.0.0 to be accessible if needed, default port 5000
+    app.run(debug=True, host='0.0.0.0', port=5050)

build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Exit on error
+set -o errexit
+
+pip install -r requirements.txt
+playwright install chromium

debug_http.html

@@ -0,0 +1 @@
+<!DOCTYPE html><html lang="en-US"><head><title>Just a moment...</title><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=Edge"><meta name="robots" content="noindex,nofollow"><meta name="viewport" content="width=device-width,initial-scale=1"><style>*{box-sizing:border-box;margin:0;padding:0}html{line-height:1.15;-webkit-text-size-adjust:100%;color:#313131;font-family:system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji"}body{display:flex;flex-direction:column;height:100vh;min-height:100vh}.main-content{margin:8rem auto;padding-left:1.5rem;max-width:60rem}@media (width <= 720px){.main-content{margin-top:4rem}}.h2{line-height:2.25rem;font-size:1.5rem;font-weight:500}@media (width <= 720px){.h2{line-height:1.5rem;font-size:1.25rem}}#challenge-error-text{background-image:url("data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIzMiIgaGVpZ2h0PSIzMiIgZmlsbD0ibm9uZSI+PHBhdGggZmlsbD0iI0IyMEYwMyIgZD0iTTE2IDNhMTMgMTMgMCAxIDAgMTMgMTNBMTMuMDE1IDEzLjAxNSAwIDAgMCAxNiAzbTAgMjRhMTEgMTEgMCAxIDEgMTEtMTEgMTEuMDEgMTEuMDEgMCAwIDEtMTEgMTEiLz48cGF0aCBmaWxsPSIjQjIwRjAzIiBkPSJNMTcuMDM4IDE4LjYxNUgxNC44N0wxNC41NjMgOS41aDIuNzgzem0tMS4wODQgMS40MjdxLjY2IDAgMS4wNTcuMzg4LjQwNy4zODkuNDA3Ljk5NCAwIC41OTYtLjQwNy45ODQtLjM5Ny4zOS0xLjA1Ny4zODktLjY1IDAtMS4wNTYtLjM4OS0uMzk4LS4zODktLjM5OC0uOTg0IDAtLjU5Ny4zOTgtLjk4NS40MDYtLjM5NyAxLjA1Ni0uMzk3Ii8+PC9zdmc+");background-repeat:no-repeat;background-size:contain;padding-left:34px}@media (prefers-color-scheme: dark){body{background-color:#222;color:#d9d9d9}}</style><meta http-equiv="refresh" content="360"></head><body><div class="main-wrapper" role="main"><div class="main-content"><noscript><div class="h2"><span id="challenge-error-text">Enable JavaScript and cookies to continue</span></div></noscript></div></div><script>(function(){window._cf_chl_opt = {cvId: '3',cZone: 'www.fbi.gov',cType: 'managed',cRay: '9c05546b1e633f68',cH: 'J0yrkRNPaDCziury9CSlfYwesCNpXp4kfPy4irDm.Ds-1768815197-1.2.1.1-q9YiSrWgTxtwx.XCt_qPc8YJxzgkXCfALBg_O0uaN1ZCE5Ze.MxpjZFgaFLPhMLY',cUPMDTk:"\/how-we-can-help-you\/victim-services?__cf_chl_tk=n6KGOAmWagfs6QF1_abE2BWYdTj2eUqXw1UwGom7jEY-1768815197-1.0.1.1-idHEIIdjaYWVO1lt5QRtWNsei_9ZLgurGHbuL02THdM",cFPWv: 'b',cITimeS: '1768815197',cTplC:0,cTplV:5,cTplB: '0',fa:"\/how-we-can-help-you\/victim-services?__cf_chl_f_tk=n6KGOAmWagfs6QF1_abE2BWYdTj2eUqXw1UwGom7jEY-1768815197-1.0.1.1-idHEIIdjaYWVO1lt5QRtWNsei_9ZLgurGHbuL02THdM",md: 'wj0rckWOoYO2WQkHlTxw1ExmOTg9NBIt0fRbvMzMw8I-1768815197-1.2.1.1-FuUpPe.mvUciLGY4rakp03B5Vsxa3rAN3.rhwvvK7KZQmscXEgw1VD_Acm1vdRIxhNyr9l4o02E2Z1KDevFXb0qukXJ.DJlLyB5fKS6MYTEkaP9dpT7eJM220EnEuUzD8gBPSXk1UW6__Iw7xhho0sAZpq.eCK6So5PN1NRTqxbz6Be3GUR4OJIoMULQlPLhvSEI42tmB_g8cibCXsR1xZtBjDXsB10DR_zIAD.NOn02SLPrWsUOE0fCtTwcc8MdEDZw1wSdWi6fkG13goTFOutuK5aPD.eb_b4mdamOrAEm3qoG42OxkXwrQB1gqb63NEaxyZdejqtO2772x4V0dbvDJ5sLtaQvleK7vpgQENG1Q03f.sLfFiI0fnhpOeAvROCtc7ow_sAAYNawKiVESJln_96MfClrzD8PUyhdCA.8nWc1ZWP.8qoQoC.8qgljsjaClSSyKvFjz9j0Mo41HADu5em5PDpeo5Y.VeFaaX6_HijDL5MkdFW.TEdy0U6_..9ZTMMComrsRSO9nUTzUZBEcdg1J9Q5JvZ2dF_TmIGJ5dvUDiuS2_qKyncU9QMstD0oiHW39vDP4OlAsYmsMvqx7OMZUcu3NWc_8RkiTfuMCzovqp66gOn9Tmh6jqpf7bx6bbkKSw_vl3hi4bKRFHqN4jT2DtMKROy5gz9YsTmtQWoMZmgv9EAVUGudQhg4xbIXyHULwVNh9oRLcsuJXmIB8AvdSZ6kjd97C3HAOWOkaIZn1YIb7UFyhHsrnBqq7lFU06OkdUnvVc3bjqv9VX.AeIR3h5ke0xO.N1GoqI3WWjpFKYeHf6JZ_nQP7m0JOaLfJ_tB7Ni2qSWEaZ1FE2QQVZN4yh_BgngRZFSNyiHvc6ytL8T.GLiFhu.IaeMYpgG7uYmqp6wmmPj.duEpLT_ttzSKJlsfjCjXri55DLlzfIfHfTdSSigKQKQEsqoBaHu85KHR13a7ZkoJmEhSBAnIGnowYa8HoxSHB0L9ldHOVfBnuq3tfMeTCmHHJ5oX9ZP3lLHIhgFeMHfyMS.9uojxpEun7vNv1kh.rYFnu2G169Sgty8uf_Ndw56GTVVKsPKoUf2TAeQUnm8vhD.drf9EAejQKp.paC.KTal1T28',mdrd: 'rSzKj773kw_pKp6Uryow95IH30LYZtHZH5zqRhYuMEs-1768815197-1.2.1.1-.2nU5JUt1K_Oy.DWKwUKZO4Kl8rDVO942Ll916fyIFt1l_aQOnJ8UDgzLDbkLe1uds_8NtD6SVT3TbXaHLN1f2Si0u2YL40nA4BWnPaf.bww65u1n8zlqBPhFazJJviMYv1na7Vifcs9DAmRAGICv36dNXMnxa0BbZVCcmxeXnVBaDej1_9pV8VZMiWrkzLTGDJi3vWZOpXt2LV6lK3ys_FGG0d7Rq2cPo2T4kdQyZ05pHAciJlIt1KcTn47n3N1yCpQoDPysyiohzlYjfX2jC6Ui6ZU0illV.dhrFu1C4HGo7_EP6iuSPH6y4e1sCi349WWcs7xpkU1FbQnOPzETRUki1mntwp0FuZbBtc0AvX82K5V_fpmYp9xSb2_6GP5YA52gHwnsvnPq.D6tCIwIrzeqsDRsw6PU55QSC8eFWd1j0hug5hIGYwkKzQRX20N7pWmGtc59t_gWpS_db.wUiE2HXlLSWbZvW8fDi283LLobk_Cpa5OosYa1guj6yjCgau8tKdbFmperQOCmhJZ4K4rTsktw4qqHSJ4f7fHa0m38VaWtnp20okJlaZXH5APc9MJv2QlD.djqZORc9lHklmfqKPdXB5wMSjslzac8zoINu8dURuUkhY8o3JXUgo83AKCf5WUSkAUBEG8ba6fFA1W6GGvyQxN.oeDxuKoMc3V2dMMDsRgGzaEaij.NWo.wxLaFFEC0EEVtMM.bKne9q1fLZ7u9.M0ZyjdzfFmb9dCNfRXzjNh8FwKssjmImGlxY50CMPgWaDK3CMSrS_aZSjs8mha_kAjq1MpINh08feCwCeiH5sXHYKxYJvxDz6VkSP5VyG8PW6P00JWY6xAiqY8csXmp2Ah.jS9x3KhUMowqhfzwWHsYriKeSGRkw5XBcqShThaskeuIfPrqmDPIWrqJtl0I3ByVZmNVnXY7.bp4i3o6xD61zb8vcTAr4XOHY.lt8ncpeNj0AfV8FTwMphc9FjggWPKK0DWyYmQ9Ah8eGPUSqfcaiJ1JdCM_BWQwJHwr8ctWjJAI1HL3PCJPz3MR55qPOVRYNqeQKhc4jA32fN9_h6ZAw5JhyRwNbws83eGQ1R2mdocBgG29NW_lOvKR0rYH9nbR3UHrqO5xo5ytha28imyfELwEyFWRU4ojms8DrLKeGgA7W3QKnlM90Z1jXjTrX0t4ypqzbbUHIUOzDlI0ibALHs92dAIb7Vi1BQj3Of1JL18u8Vn7ZqTT4MHrq8M7NFAvb12vW7cd2tSnfUPp2MM4Y1jvw75jibtZv0l6dFdcXd9xHVcJ3FhIlzfwBqT.K1FHHqJvEm59xMvL4HbHu0I0pdrPuUXkqsUo2Q2q4xEaMcZscdoLp7JKWYILfdQCP9dQIaR0pXSBMpM3snUpNH.GIyq08qev699t26KiN2L9TXfmNnizIoHj9viS1CToZ2FvJHL4HNNfHDdVbNzrBUS6LgCxSvtBql8M4ATuOFV.4QgbKT2O3Xew7g3V9BtCwfmLx7A3F4X9rgBjtirRTeuL.nFqd3WOnSh_phGEhm.8n8Fku5WcKiL9vQoOFvPNgAoq4qBgQxB254puHzDMEVGs_eGljrO87KDBaYGhFjj7gEwUy62GzZdkuW_ryolkHcIyrXMlzSIgQ5KLSZ_XH7yYA..QXMiJP8p4ju0Iy_6ycAW8EHzbH2LbjD46i9NobQHImqDtFHFE2oEZWu0UeA5ILk4dn8.5Zm4G3TBDCx1ETi4nKRFqW0nk6gOmvQKSVTmQP4KldusuJfvHWoImT_zzBQ.uLkaKc.dnBp3VWt.GJF_qWtEbGG7L75dypkzkhrS.IkajYZFOQMeEkdKPwYgcEU8RnstkMDoAN9tXn.DMpDx22Yun50j3R1j9CRRq9UMLswUX2LZygoM5vIWVDw9S_.EL5tMCJp_NSYP8NY3k_JWaZtAnnJMqBRVu_Qr0nhwYlzGK_tgkNmsMHGtU499uE58ZsmgA3e.iQ3ADmYtX9v_5t7TTtMf6HO8gOANHxBmMpI4KkMMr6FvR6DswYSnAPuNSEFz7UKkxfnAxTJa9G.pNwP.Qsr3iWlz1z_5xrVej1Rwq2sRn2xhGRq6Ud8ctFa.2vb4AaA6o4gD_.3YWb41oalPDTkUZBEjXMm2gTnehKwv3Q3JMYUU1nIODJwu_IOryVgOaHKACjOiNaaJ3DgHJu9WA839m3_fZYShU6SxIX2TWTZBwDmhyYsuVGnqswNTmbIwf1P5g6LzUn.izvrUfLNtqeZ5qqj3k0UbdOqDnZ5h0c5dP5r0UdTMBMmid26Ji47SuK0eu4E1R2T4FqgPOHc.Phy7.SUJE0d5FSKpoBklI_d5AwZg1Mzuw.FhsfrigHo0KDlk6bbDpk348x3Yy5iMmEzqfpRw_0ufJgaRpT88yUM9j3lOAQeEhL4jTEsdO_GpI8rj7ZF5sYPnAeHHxLXQH6m9vs7x40RS8k5cYxliKwlhWdZARRr11Cjk0gsOnS.DSty8Y3gYnvv02yEWY2yxG.RfyVdewVRDHnf7XWQ7mnpGOg5VG6n5RaSvsjZLHQn66kQC0l0vC8FZX0zBRMmUIOPnk7uX.0zzjhxTweuxni8CoMYmVV6HRyKHiGFTscB5zWmJIE8NNuL8Yf64ikt_VRGiMrHqg8H3z.usm3or5fS9UBlvpqv36H0fzEMfJMqtjbo.i2IUK2kVsvnWPoo3nVQcPqnJqKp35q1MItpVhAzRE4I',};var a = document.createElement('script');a.src = '/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page/v1?ray=9c05546b1e633f68';window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, location.href.length - window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;if (window.history && window.history.replaceState) {var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;history.replaceState(null, null,"\/how-we-can-help-you\/victim-services?__cf_chl_rt_tk=n6KGOAmWagfs6QF1_abE2BWYdTj2eUqXw1UwGom7jEY-1768815197-1.0.1.1-idHEIIdjaYWVO1lt5QRtWNsei_9ZLgurGHbuL02THdM"+ window._cf_chl_opt.cOgUHash);a.onload = function() {history.replaceState(null, null, ogU);}}document.getElementsByTagName('head')[0].appendChild(a);}());</script></body></html>

main.py

@@ -0,0 +1,63 @@
+import argparse
+import sys
+from src.crawler import Crawler
+from src.analyzer import ImageAnalyzer
+from src.reporter import generate_json_report, generate_html_report
+
+def main():
+    parser = argparse.ArgumentParser(description="Web Image Alt Text Checker (Whole Domain)")
+    parser.add_argument("url", help="The Start URL of the domain to crawl (e.g. https://example.com)")
+    parser.add_argument("--output", choices=['json', 'html', 'both'], default='html', help="Report output format")
+    parser.add_argument("--limit", type=int, default=100, help="Max pages to crawl (default 100)")
+    args = parser.parse_args()
+
+    # Crawl
+    # Crawl
+    print(f"Starting domain crawl from: {args.url}")
+    crawler = Crawler()
+    site_data, total_discovered, blocked_reason = crawler.crawl_domain(args.url, max_pages=args.limit)
+    
+    if not site_data and blocked_reason:
+        print(f"\n[!] CRAWL BLOCKED: {blocked_reason}")
+        # Proceed to generate report even if empty to show the block status
+    elif not site_data:
+        print("No data crawled. Please check the URL and try again.")
+        sys.exit(1)
+
+    print(f"Crawl complete. Scanned {len(site_data)} pages.")
+
+    # Analyze
+    print("Analyzing images for missing alt text...")
+    analyzer = ImageAnalyzer()
+    results = analyzer.analyze_site(site_data)
+    
+    # Add discovery stats to summary
+    results['summary']['total_pages_discovered'] = total_discovered
+    results['summary']['blocked_reason'] = blocked_reason
+    results['summary']['crawl_blocked'] = bool(blocked_reason)
+
+    # Report
+    print("Generating reports...")
+    if args.output in ['json', 'both']:
+        generate_json_report(results, "seo_report.json")
+    if args.output in ['html', 'both']:
+        generate_html_report(results, "seo_report.html")
+
+    summary = results['summary']
+    print("\n--- Summary ---")
+    if summary.get('crawl_blocked'):
+        print(f"STATUS: BLOCKED ({summary['blocked_reason']})")
+        print("Results are likely incomplete.")
+        
+    print(f"Total Pages Discovered: {summary['total_pages_discovered']}")
+    print(f"Pages Scanned: {summary['total_pages_scanned']}")
+    print(f"Total Images: {summary['total_images_found']}")
+    print(f"Images Missing Alt: {summary['total_images_missing_alt']}")
+    
+    if summary['total_images_missing_alt'] > 0:
+        print("Please check the generated report for details.")
+    elif not summary.get('crawl_blocked'):
+        print("All images have alt text! Great job.")
+
+if __name__ == "__main__":
+    main()

qodana.yaml

@@ -0,0 +1,23 @@
+#-------------------------------------------------------------------------------#
+#               Qodana analysis is configured by qodana.yaml file               #
+#             https://www.jetbrains.com/help/qodana/qodana-yaml.html            #
+#-------------------------------------------------------------------------------#
+version: "1.0"
+#Specify inspection profile for code analysis
+profile:
+  name: qodana.starter
+#Enable inspections
+#include:
+#  - name: <SomeEnabledInspectionId>
+#Disable inspections
+#exclude:
+#  - name: <SomeDisabledInspectionId>
+#    paths:
+#      - <path/where/not/run/inspection>
+#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
+#bootstrap: sh ./prepare-qodana.sh
+#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
+#plugins:
+#  - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
+#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
+linter: jetbrains/qodana-<linter>:2024.3

requirements.txt

@@ -0,0 +1,10 @@
+requests
+beautifulsoup4
+Pillow
+fake-useragent
+cloudscraper
+curl_cffi
+flask
+flask-cors
+playwright
+gunicorn

seo_report.json

@@ -0,0 +1,227 @@
+{
+    "summary": {
+        "total_pages_scanned": 25,
+        "total_images_found": 233,
+        "total_images_missing_alt": 68,
+        "total_pages_discovered": 301,
+        "blocked_reason": null,
+        "crawl_blocked": false
+    },
+    "details": [
+        {
+            "page_url": "https://www.fbi.gov/",
+            "missing_alt_count": 2,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/history",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/how-we-can-help-you/active-shooter-safety-resources",
+            "missing_alt_count": 0,
+            "images_without_alt": []
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/can-i-obtain-detailed-information-about-a-current-fbi-investigation-that-i-see-in-the-news",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/investigate/violent-crime",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/what-career-paths-does-the-fbi-offer-besides-the-special-agent-role",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/how-we-can-help-you/victim-services",
+            "missing_alt_count": 0,
+            "images_without_alt": []
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/what-is-the-fbis-foreign-counterintelligence-responsibility",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/has-the-ending-of-the-cold-war-reduced-the-amount-of-espionage",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/what-is-the-fbis-responsibility-in-bombing-cases",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://artcrimes.fbi.gov",
+            "missing_alt_count": 7,
+            "images_without_alt": [
+                "https://artcrimes.fbi.gov/image-repository/nsaf_slide2.jpg/@@images/image/large",
+                "https://artcrimes.fbi.gov/image-repository/nsaf_slide1.jpg/@@images/image/large",
+                "https://artcrimes.fbi.gov/++theme++fbinsaf.theme/img/assets/logo/banner_1295x200.png",
+                "https://artcrimes.fbi.gov/++theme++fbinsaf.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://artcrimes.fbi.gov/++theme++fbinsaf.theme/img/assets/logo/fbi-seal.png",
+                "https://artcrimes.fbi.gov/++theme++fbinsaf.theme/uswds-2.9.0/img/icon-https.svg",
+                "https://artcrimes.fbi.gov/image-repository/nsaf_slide3.jpg/@@images/image/large"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/investigate/organized-crime",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/do-fbi-agents-work-with-state-local-or-other-law-enforcement-officers-on-task-forces",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/investigate",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/what-is-the-critical-incident-response-group",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/history/history-stories--multimedia",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/history/wall-of-honor/jimmie-john-daniels",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/investigate/violent-crime/gangs",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/what-does-the-fbi-do-with-information-and-evidence-gathered-during-an-investigation",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/contact-us/field-offices/baltimore/news/ms-13-clique-leader-indicted-in-connection-with-racketeering-conspiracy-involving-four-murders-and-two-attempted-murders-in-baltimore",
+            "missing_alt_count": 0,
+            "images_without_alt": []
+        },
+        {
+            "page_url": "https://www.fbi.gov/news/espanol",
+            "missing_alt_count": 2,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/about/faqs/when-was-the-fbi-founded",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/image-repository/fbi-nasa-flag-certificate-070522.jpg/view",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        },
+        {
+            "page_url": "https://www.fbi.gov/news/stories/events-mark-the-50th-anniversary-of-female-special-agents-in-the-fbi",
+            "missing_alt_count": 3,
+            "images_without_alt": [
+                "https://www.fbi.gov/++theme++fbigov.theme/images/fbi_seal_new.png",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-dot-gov.svg",
+                "https://www.fbi.gov/++theme++fbigov.theme/uswds-2.9.0/img/icon-https.svg"
+            ]
+        }
+    ]
+}

src/analyzer.py

@@ -0,0 +1,41 @@
+
+class ImageAnalyzer:
+    def analyze_site(self, site_data):
+        """
+        Analyzes the crawled site data for missing ALT text.
+        :param site_data: Dict { page_url: [image_dicts] }
+        :return: Dict containing summary and detailed report.
+        """
+        total_images = 0
+        total_missing_alt = 0
+        pages_report = []
+
+        for page_url, images in site_data.items():
+            missing_images = []
+            for img in images:
+                total_images += 1
+                alt_text = img.get('alt')
+                
+                # Check for strictly missing alt text (None or empty string)
+                if not alt_text or alt_text.strip() == "":
+                    # Double check if 'alt' attribute existed but was empty vs didn't exist?
+                    # The crawler uses .get('alt', '') so it's '' if missing.
+                    # User asked for "missing alt text".
+                    # We will count empty string as missing alt text for SEO purposes.
+                    missing_images.append(img['src'])
+                    total_missing_alt += 1
+            
+            pages_report.append({
+                "page_url": page_url,
+                "missing_alt_count": len(missing_images),
+                "images_without_alt": list(set(missing_images))
+            })
+
+        return {
+            "summary": {
+                "total_pages_scanned": len(site_data),
+                "total_images_found": total_images,
+                "total_images_missing_alt": total_missing_alt
+            },
+            "details": pages_report
+        }

src/crawler.py

@@ -0,0 +1,276 @@
+from curl_cffi import requests
+import time
+import random
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+import sys
+import logging
+import os
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+class Crawler:
+    def __init__(self, use_playwright=True):
+        # 1. Improved HTTP Client Setup
+        self.session = requests.Session(impersonate="chrome120")
+        self.session.headers.update({
+            "Referer": "https://www.google.com/",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-User": "?1",
+            "Sec-Fetch-Dest": "document",
+        })
+        self.visited_urls = set()
+        self.use_playwright_fallback = use_playwright
+        self.blocked_reason = None
+
+    def fetch(self, url):
+        """
+        Orchestrates fetching: tries HTTP first, detects WAF/Challenge, falls back to Playwright if needed.
+        """
+        content = self._fetch_http(url)
+        
+        # WAF/Challenge Detection
+        # Key terms often found in block pages
+        block_keywords = [
+            "api-services-support@amazon.com", 
+            "robot check", 
+            "type the characters you see", 
+            "challenge.js", 
+            "verify you are human",
+            "captcha",
+            "incident id"
+        ]
+        
+        is_blocked = False
+        if content:
+             lower_content = content.lower()
+             if any(k in lower_content for k in block_keywords):
+                 is_blocked = True
+        elif content is None:
+             # If HTTP failed completely (e.g. 403 or connection error), we treat it as potentially blocked if it's high-profile
+             is_blocked = True
+
+        if is_blocked:
+            logger.warning(f"WAF/CAPTCHA or block detected for {url} using HTTP client.")
+            self.blocked_reason = "WAF/JS challenge/consent"
+            
+            if self.use_playwright_fallback:
+                logger.info("Attempting fallback to Playwright (Browser Automation)...")
+                return self._fetch_playwright(url)
+            else:
+                logger.error("Playwright fallback disabled. Skipping page.")
+                return None
+        
+        return content
+
+    def _fetch_http(self, url):
+        """
+        Standard robust HTTP fetch with retry and backoff.
+        """
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                response = self.session.get(url, timeout=30, allow_redirects=True)
+                
+                # Debug logging for response
+                logger.info(f"HTTP GET {url} - Status: {response.status_code} - URL: {response.url}")
+                
+                # Check for 200 OK (or close to it)
+                if 200 <= response.status_code < 300:
+                    return response.text
+                
+                # If 403, it might be WAF. Return text so we can check for challenge keywords.
+                if response.status_code == 403:
+                    # Save HTML for debugging
+                    with open("debug_http.html", "w", encoding="utf-8") as f:
+                        f.write(response.text)
+                    logger.info("Saved 403 response body to debug_http.html")
+                    return response.text 
+                
+                if response.status_code in [429, 500, 502, 503]:
+                    logger.warning(f"Status {response.status_code} for {url}. Retrying ({attempt + 1}/{max_retries})...")
+                    time.sleep(2 * (attempt + 1))
+                    continue
+
+                logger.error(f"Error fetching {url}: Status {response.status_code}")
+                return None
+
+            except Exception as e:
+                logger.error(f"Exception fetching {url}: {e}. Retrying...")
+                time.sleep(2 * (attempt + 1))
+        
+        return None
+
+    def _fetch_playwright(self, url):
+        """
+        Fallback Strategy A: Use Playwright to render full page.
+        """
+        try:
+            from playwright.sync_api import sync_playwright
+        except ImportError:
+            logger.error("Playwright not installed. Run `pip install playwright` and `playwright install`.")
+            return None
+
+        try:
+            with sync_playwright() as p:
+                # Launch options
+                # headless=False can be useful for visual debugging if needed, but we keep True for automation.
+                browser = p.chromium.launch(headless=True)
+                
+                # Context with realistic viewport and locale
+                context = browser.new_context(
+                    viewport={'width': 1920, 'height': 1080},
+                    user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                    locale="en-US",
+                    timezone_id="America/New_York",
+                    java_script_enabled=True
+                )
+                
+                page = context.new_page()
+                try:
+                    logger.info(f"Playwright navigating to {url}...")
+                    page.goto(url, timeout=60000, wait_until="domcontentloaded")
+                    
+                    # Wait for a bit of hydration
+                    page.wait_for_timeout(3000)
+                    
+                    # Log page title
+                    title = page.title()
+                    logger.info(f"Page Title: {title}")
+                    
+                    # CAPTCHA Detection via Selectors
+                    # Amazon often uses #captchacharacters
+                    if page.locator("input#captchacharacters").count() > 0 or "Robot Check" in title:
+                        logger.error("Playwright hit visible CAPTCHA. Manual intervention required.")
+                        self.blocked_reason = "Hard Catcha/Robot Check"
+                        
+                        # Save screenshot for proof
+                        page.screenshot(path="amazon.png")
+                        logger.info("Saved screenshot to amazon.png")
+                        
+                        # Save HTML
+                        with open("debug_playwright.html", "w", encoding="utf-8") as f:
+                            f.write(page.content())
+                        
+                        return None # Return None to signify blocked/useless content
+
+                    content = page.content()
+                    return content
+                    
+                except Exception as nav:
+                     logger.error(f"Playwright navigation error: {nav}")
+                     return None
+                finally:
+                    browser.close()
+
+        except Exception as e:
+            logger.error(f"Playwright critical error: {e}")
+            return None
+
+    def crawl_domain(self, start_url, max_pages=100):
+        """
+        Crawls domain with robust link discovery and normalization.
+        """
+        # Reset blocked state
+        self.blocked_reason = None
+        
+        parsed_start = urlparse(start_url)
+        start_domain = parsed_start.netloc
+        base_domain = start_domain.replace('www.', '') # simplistic base domain
+        
+        queue = [start_url]
+        self.visited_urls.add(start_url)
+        
+        site_data = {}
+        pages_crawled = 0
+
+        logger.info(f"Starting crawl for domain: {base_domain}")
+
+        while queue and pages_crawled < max_pages:
+            pop_index = random.randint(0, len(queue) - 1)
+            current_url = queue.pop(pop_index)
+
+            logger.info(f"Crawling [{pages_crawled + 1}]: {current_url}")
+            
+            html_content = self.fetch(current_url)
+            
+            # If fetch failed or blocked
+            if not html_content:
+                if self.blocked_reason and pages_crawled == 0:
+                    logger.error("Crawl blocked on first page. Aborting.")
+                    break
+                continue
+
+            # Extract images
+            images = self.extract_images(html_content, current_url)
+            site_data[current_url] = images
+            pages_crawled += 1
+
+            # --- Robust Link Discovery ---
+            soup = BeautifulSoup(html_content, 'html.parser')
+            links_found = soup.find_all('a', href=True)
+            
+            links_stats = {"total": len(links_found), "kept": 0, "skipped": 0}
+            
+            for link in links_found:
+                href = link['href']
+                
+                # Resolve relative URL
+                full_url = urljoin(current_url, href)
+                parsed_url = urlparse(full_url)
+                
+                # Normalize: remove fragment
+                full_url = parsed_url._replace(fragment="").geturl()
+                
+                link_domain = parsed_url.netloc
+                
+                # Domain Check (Allow www. and non-www.)
+                is_internal = link_domain == start_domain or link_domain.endswith('.' + base_domain) or link_domain == base_domain
+                
+                if is_internal:
+                    # Filter non-html resources
+                    path = parsed_url.path.lower()
+                    excluded_exts = ['.jpg', '.jpeg', '.png', '.gif', '.css', '.js', '.ico', '.svg', '.pdf', '.zip', '.xml']
+                    
+                    if any(path.endswith(ext) for ext in excluded_exts):
+                         links_stats["skipped"] += 1
+                         continue
+
+                    if full_url not in self.visited_urls:
+                        self.visited_urls.add(full_url)
+                        queue.append(full_url)
+                        links_stats["kept"] += 1
+                    else:
+                        links_stats["skipped"] += 1 # Already visited
+                else:
+                    links_stats["skipped"] += 1 # External
+
+            logger.info(f"Link Discovery: Found {links_stats['total']}, Added {links_stats['kept']} new unique internal links.")
+        
+        # Final block check: if we scraped 0 pages or only 1 page with 0 images and blocked_reason is set
+        if pages_crawled == 0 and self.blocked_reason:
+             pass # blocked_reason is already set
+        elif pages_crawled > 0 and not self.blocked_reason:
+             # Check if we successfully scanned but found suspiciously nothing?
+             pass
+
+        return site_data, len(self.visited_urls), self.blocked_reason
+
+    def extract_images(self, html_content, base_url):
+        if not html_content:
+            return []
+        soup = BeautifulSoup(html_content, 'html.parser')
+        images = []
+        for img in soup.find_all('img'):
+            raw_src = img.get('src')
+            if not raw_src:
+                continue
+            full_url = urljoin(base_url, raw_src)
+            images.append({'src': full_url, 'alt': img.get('alt', '')})
+        return images

src/reporter.py

@@ -0,0 +1,115 @@
+import json
+import os
+
+def generate_json_report(data, output_file="report.json"):
+    """
+    Generates a JSON report of the site analysis.
+    """
+    try:
+        with open(output_file, 'w') as f:
+            json.dump(data, f, indent=4)
+        print(f"JSON report generated at {os.path.abspath(output_file)}")
+    except Exception as e:
+        print(f"Error generating JSON report: {e}")
+
+def generate_html_report(data, output_file="report.html"):
+    """
+    Generates an HTML report for the domain scan.
+    """
+    summary = data['summary']
+    details = data['details']
+
+    html = f"""
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <title>SEO Image Alt Text Report</title>
+        <style>
+            body {{ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; margin: 0; padding: 20px; background-color: #f4f6f9; }}
+            h1, h2 {{ color: #333; }}
+            .container {{ max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; box-shadow: 0 2px 5px rgba(0,0,0,0.1); }}
+            .summary-box {{ display: flex; gap: 20px; margin-bottom: 30px; }}
+            .card {{ flex: 1; padding: 20px; border-radius: 8px; color: white; text-align: center; }}
+            .bg-blue {{ background-color: #007bff; }}
+            .bg-green {{ background-color: #28a745; }}
+            .bg-red {{ background-color: #dc3545; }}
+            .card h3 {{ margin: 0; font-size: 3em; }}
+            .card p {{ margin: 0; opacity: 0.9; }}
+            
+            table {{ border-collapse: collapse; width: 100%; margin-top: 10px; }}
+            th, td {{ border: 1px solid #dee2e6; padding: 12px; text-align: left; }}
+            th {{ background-color: #f8f9fa; color: #495057; }}
+            tr:nth-child(even) {{ background-color: #f9f9f9; }}
+            .page-section {{ margin-bottom: 40px; border-top: 2px solid #eee; padding-top: 20px; }}
+            .page-url {{ font-size: 1.2em; font-weight: bold; color: #0056b3; margin-bottom: 10px; display: block; }}
+            .no-issues {{ color: #28a745; font-style: italic; }}
+            .img-link {{ word-break: break-all; color: #555; }}
+        </style>
+    </head>
+    <body>
+    <div class="container">
+        <h1>SEO Image Alt Text Report</h1>
+        
+        <div class="summary-box">
+            <div class="card bg-blue">
+                <h3>{summary['total_pages_scanned']}</h3>
+                <p>Pages Scanned</p>
+            </div>
+            <div class="card bg-green">
+                <h3>{summary['total_images_found']}</h3>
+                <p>Total Images Found</p>
+            </div>
+            <div class="card bg-red">
+                <h3>{summary['total_images_missing_alt']}</h3>
+                <p>Images Missing Alt Text</p>
+            </div>
+        </div>
+    """
+
+    if summary.get('crawl_blocked'):
+        html += f"""
+        <div style="background-color: #fff3cd; color: #856404; padding: 15px; border: 1px solid #ffeeba; border-radius: 5px; margin-bottom: 20px;">
+            <strong>⚠️ CRAWL BLOCKED:</strong> The crawler was blocked by the site ({summary.get('blocked_reason')}). Results are likely incomplete.
+            <p>Try using a residential proxy or a headless browser with stealth plugins to bypass this restriction.</p>
+        </div>
+        """
+    
+    html += """
+        <h2>Detailed Breakdown</h2>
+    """
+
+    if not details:
+        html += "<p class='no-issues'>No pages found with missing alt text images! Great job.</p>"
+    
+    for page in details:
+        html += f"""
+        <div class="page-section">
+            <a href="{page['page_url']}" class="page-url" target="_blank">Page: {page['page_url']}</a>
+            <p><strong>{page['missing_alt_count']}</strong> images missing alt text</p>
+            <table>
+                <tr><th>Image Source URL</th></tr>
+        """
+        for img_src in page['images_without_alt']:
+            html += f"""
+                <tr>
+                    <td><a href="{img_src}" target="_blank" class="img-link">{img_src}</a></td>
+                </tr>
+            """
+        html += "</table></div>"
+
+    html += """
+    </div>
+    </body>
+    </html>
+    """
+    
+    try:
+        with open(output_file, 'w') as f:
+            f.write(html)
+        print(f"HTML report generated at {os.path.abspath(output_file)}")
+    except Exception as e:
+        print(f"Error generating HTML report: {e}")
+
+
+
+