Add unit tests for sitemap parser and URL extraction functions

docs/sample_urlscan_result.json

@@ -0,0 +1,488 @@
+{
+  "data": {
+    "requests": [
+      {
+        "request": {
+          "requestId": "FE77961E144ADAA2A7D7760E8CE7EB6A",
+          "loaderId": "FE77961E144ADAA2A7D7760E8CE7EB6A",
+          "documentURL": "https://example.com/",
+          "request": {
+            "url": "https://example.com/",
+            "method": "GET",
+            "headers": {
+              "Upgrade-Insecure-Requests": "1",
+              "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"
+            },
+            "mixedContentType": "none",
+            "initialPriority": "VeryHigh",
+            "referrerPolicy": "strict-origin-when-cross-origin",
+            "isSameSite": true
+          },
+          "timestamp": 34065601.895598,
+          "wallTime": 1766933427.494116,
+          "initiator": {
+            "type": "other"
+          },
+          "redirectHasExtraInfo": false,
+          "type": "Document",
+          "frameId": "61B9C1B13CBD0AC8E19A4CB13D24DE8E",
+          "hasUserGesture": false,
+          "primaryRequest": true
+        },
+        "response": {
+          "encodedDataLength": 536,
+          "dataLength": 513,
+          "requestId": "FE77961E144ADAA2A7D7760E8CE7EB6A",
+          "type": "Document",
+          "response": {
+            "url": "https://example.com/",
+            "status": 200,
+            "statusText": "",
+            "headers": {
+              "age": "13121",
+              "allow": "GET, HEAD",
+              "cf-cache-status": "HIT",
+              "cf-ray": "9b51dec2092c3738-FRA",
+              "content-encoding": "gzip",
+              "content-type": "text/html",
+              "date": "Sun, 28 Dec 2025 14:50:27 GMT",
+              "last-modified": "Sat, 20 Dec 2025 05:43:40 GMT",
+              "server": "cloudflare",
+              "vary": "Accept-Encoding"
+            },
+            "mimeType": "text/html",
+            "charset": "",
+            "remoteIPAddress": "104.18.27.120",
+            "remotePort": 443,
+            "encodedDataLength": 536,
+            "timing": {
+              "requestTime": 34065601.895873,
+              "proxyStart": -1,
+              "proxyEnd": -1,
+              "dnsStart": 0,
+              "dnsEnd": 5.906,
+              "connectStart": 5.942,
+              "connectEnd": 25.448,
+              "sslStart": 11.119,
+              "sslEnd": 25.442,
+              "workerStart": -1,
+              "workerReady": -1,
+              "workerFetchStart": -1,
+              "workerRespondWithSettled": -1,
+              "sendStart": 25.509,
+              "sendEnd": 25.562,
+              "pushStart": 0,
+              "pushEnd": 0,
+              "receiveHeadersStart": 83.246,
+              "receiveHeadersEnd": 84.37
+            },
+            "responseTime": 1766933427577.645,
+            "protocol": "h2",
+            "alternateProtocolUsage": "unspecifiedReason",
+            "securityState": "secure",
+            "securityDetails": {
+              "protocol": "TLS 1.3",
+              "keyExchange": "",
+              "keyExchangeGroup": "X25519MLKEM768",
+              "cipher": "AES_128_GCM",
+              "certificateId": 0,
+              "subjectName": "example.com",
+              "sanList": [
+                "example.com",
+                "*.example.com"
+              ],
+              "issuer": "Cloudflare TLS Issuing ECC CA 3",
+              "validFrom": 1765913972,
+              "validTo": 1773685964,
+              "signedCertificateTimestampList": [
+                {
+                  "status": "Verified",
+                  "origin": "Embedded in certificate",
+                  "logDescription": "DigiCert 'Wyvern2026h1'",
+                  "logId": "6411C46CA412ECA7891CA2022E00BCAB4F2807D41E3527ABEAFED503C97DCDF0",
+                  "timestamp": 1765914573355,
+                  "hashAlgorithm": "SHA-256",
+                  "signatureAlgorithm": "ECDSA",
+                  "signatureData": "304502207EAD9214E5E3A94FD2FFFC3AACE657456FE8357607018CE2D8B75E80D14359C5022100F3A4736BB259749EB4EBF22271E5F29F15688F2193F659FC0AE7A3C2A9C4BF41"
+                },
+                {
+                  "status": "Verified",
+                  "origin": "Embedded in certificate",
+                  "logDescription": "Cloudflare 'Nimbus2026'",
+                  "logId": "CB38F715897C84A1445F5BC1DDFBC96EF29A59CD470A690585B0CB14C31458E7",
+                  "timestamp": 1765914573388,
+                  "hashAlgorithm": "SHA-256",
+                  "signatureAlgorithm": "ECDSA",
+                  "signatureData": "304402207CB577326AA9CEE312D5B03DE9930181A98958F77CB911850DF269CD39D1A09802201A31F3B9B9AA2A3010320E38E619B3EE5D1F998FED2CAEC05CB9EBFEE5C1C36A"
+                },
+                {
+                  "status": "Verified",
+                  "origin": "TLS extension",
+                  "logDescription": "Google 'Xenon2026h1' log",
+                  "logId": "969764BF555897ADF743876837084277E9F03AD5F6A4F3366E46A43F0FCAA9C6",
+                  "timestamp": 1765914576543,
+                  "hashAlgorithm": "SHA-256",
+                  "signatureAlgorithm": "ECDSA",
+                  "signatureData": "304402203CB5BE0CE4168D2F0F81BC896F518D7FF7EFF8B7BA72448C6B7C46DC6056CB15022055BAA26FBD349EADFA9B314204B5BA049E669BDDF386CA4F2B126BE27EDA183A"
+                },
+                {
+                  "status": "Verified",
+                  "origin": "TLS extension",
+                  "logDescription": "Google 'Argon2026h1' log",
+                  "logId": "0E5794BCF3AEA93E331B2C9907B3F790DF9BC23D713225DD21A925AC61C54E21",
+                  "timestamp": 1765914576444,
+                  "hashAlgorithm": "SHA-256",
+                  "signatureAlgorithm": "ECDSA",
+                  "signatureData": "3046022100A9914FC1A2559979D8351623D4CD7C8B3D52F29F9C3E816E59E0E7617DF21E9F022100B34E0A80A3547FF2DEEBE9A3A3E33ED211C23255A03A2F415750DCB2B02BC424"
+                }
+              ],
+              "certificateTransparencyCompliance": "compliant",
+              "serverSignatureAlgorithm": 1027,
+              "encryptedClientHello": false
+            },
+            "isIpProtectionUsed": false
+          },
+          "hasExtraInfo": true,
+          "hash": "6f5635035f36ad500b4fc4bb7816bb72ef5594e1bcae44fa074c5e988fc4c0fe",
+          "size": 513,
+          "asn": {
+            "ip": "104.18.27.120",
+            "asn": "13335",
+            "country": "US",
+            "description": "CLOUDFLARENET, US",
+            "name": "CLOUDFLARENET",
+            "route": "104.16.0.0/14"
+          },
+          "geoip": {
+            "country": "",
+            "region": "",
+            "timezone": "",
+            "city": "",
+            "ll": [
+              null,
+              null
+            ],
+            "country_name": "Ascension Island"
+          }
+        }
+      }
+    ],
+    "cookies": [],
+    "console": [],
+    "links": [
+      {
+        "href": "https://iana.org/domains/example",
+        "text": "Learn more",
+        "frameId": "61B9C1B13CBD0AC8E19A4CB13D24DE8E"
+      }
+    ],
+    "timing": {
+      "beginNavigation": "2025-12-28T14:50:27.492Z",
+      "frameStartedLoading": "2025-12-28T14:50:27.493Z",
+      "frameNavigated": "2025-12-28T14:50:27.581Z",
+      "domContentEventFired": "2025-12-28T14:50:27.593Z",
+      "frameStoppedLoading": "2025-12-28T14:50:27.595Z"
+    },
+    "globals": []
+  },
+  "lists": {
+    "ips": [
+      "104.18.27.120"
+    ],
+    "countries": [],
+    "asns": [
+      "13335"
+    ],
+    "domains": [
+      "example.com"
+    ],
+    "servers": [
+      "cloudflare"
+    ],
+    "urls": [
+      "https://example.com/"
+    ],
+    "linkDomains": [
+      "iana.org"
+    ],
+    "certificates": [
+      {
+        "subjectName": "example.com",
+        "issuer": "Cloudflare TLS Issuing ECC CA 3",
+        "validFrom": 1765913972,
+        "validTo": 1773685964
+      }
+    ],
+    "hashes": [
+      "6f5635035f36ad500b4fc4bb7816bb72ef5594e1bcae44fa074c5e988fc4c0fe"
+    ]
+  },
+  "meta": {
+    "processors": {
+      "umbrella": {
+        "data": [
+          {
+            "hostname": "example.com",
+            "rank": 6940
+          }
+        ]
+      },
+      "geoip": {
+        "data": [
+          {
+            "ip": "104.18.27.120",
+            "geoip": {
+              "country": "",
+              "region": "",
+              "timezone": "",
+              "city": "",
+              "ll": [
+                null,
+                null
+              ],
+              "country_name": "Ascension Island"
+            }
+          }
+        ]
+      },
+      "asn": {
+        "data": [
+          {
+            "ip": "104.18.27.120",
+            "asn": "13335",
+            "country": "US",
+            "description": "CLOUDFLARENET, US",
+            "name": "CLOUDFLARENET",
+            "route": "104.16.0.0/14"
+          }
+        ]
+      },
+      "rdns": {
+        "data": []
+      },
+      "wappa": {
+        "data": []
+      }
+    }
+  },
+  "page": {
+    "asn": "AS13335",
+    "asnname": "CLOUDFLARENET, US",
+    "city": "",
+    "country": "",
+    "domain": "example.com",
+    "ip": "104.18.27.120",
+    "server": "cloudflare",
+    "url": "https://example.com/",
+    "apexDomainAgeDays": 4034,
+    "language": "en",
+    "mimeType": "text/html",
+    "title": "Example Domain",
+    "tlsValidDays": 89,
+    "tlsAgeDays": 11,
+    "domainAgeDays": 4034,
+    "tlsValidFrom": "2025-12-16T19:39:32.000Z",
+    "umbrellaRank": 6940,
+    "apexDomain": "example.com",
+    "tlsIssuer": "Cloudflare TLS Issuing ECC CA 3",
+    "status": "200"
+  },
+  "scanner": {
+    "country": "de"
+  },
+  "stats": {
+    "IPv6Percentage": 0,
+    "adBlocked": 0,
+    "domainStats": [
+      {
+        "count": 1,
+        "ips": [
+          "104.18.27.120"
+        ],
+        "domain": "example.com",
+        "size": 513,
+        "encodedSize": 536,
+        "countries": [],
+        "index": 0,
+        "initiators": [],
+        "redirects": 0
+      }
+    ],
+    "ipStats": [
+      {
+        "requests": 1,
+        "domains": [
+          "example.com"
+        ],
+        "ip": "104.18.27.120",
+        "asn": {
+          "ip": "104.18.27.120",
+          "asn": "13335",
+          "country": "US",
+          "description": "CLOUDFLARENET, US",
+          "name": "CLOUDFLARENET",
+          "route": "104.16.0.0/14"
+        },
+        "dns": {},
+        "geoip": {
+          "country": "",
+          "region": "",
+          "timezone": "",
+          "city": "",
+          "ll": [
+            null,
+            null
+          ],
+          "country_name": "Ascension Island"
+        },
+        "size": 513,
+        "encodedSize": 536,
+        "countries": [],
+        "index": 0,
+        "ipv6": false,
+        "redirects": 0,
+        "count": null
+      }
+    ],
+    "malicious": 0,
+    "protocolStats": [
+      {
+        "count": 1,
+        "size": 513,
+        "encodedSize": 536,
+        "ips": [
+          "104.18.27.120"
+        ],
+        "countries": [],
+        "securityState": {},
+        "protocol": "h2"
+      }
+    ],
+    "regDomainStats": [
+      {
+        "count": 1,
+        "ips": [
+          "104.18.27.120"
+        ],
+        "regDomain": "example.com",
+        "size": 513,
+        "encodedSize": 536,
+        "countries": [],
+        "index": 0,
+        "subDomains": [
+          {
+            "domain": "",
+            "country": ""
+          }
+        ],
+        "redirects": 0
+      }
+    ],
+    "resourceStats": [
+      {
+        "count": 1,
+        "size": 513,
+        "encodedSize": 536,
+        "latency": 0,
+        "countries": [],
+        "ips": [
+          "104.18.27.120"
+        ],
+        "type": "Document",
+        "compression": "1.0",
+        "percentage": 100
+      }
+    ],
+    "securePercentage": 100,
+    "secureRequests": 1,
+    "serverStats": [
+      {
+        "count": 1,
+        "size": 513,
+        "encodedSize": 536,
+        "ips": [
+          "104.18.27.120"
+        ],
+        "countries": [],
+        "server": "cloudflare"
+      }
+    ],
+    "tlsStats": [
+      {
+        "count": 1,
+        "size": 513,
+        "encodedSize": 536,
+        "ips": [
+          "104.18.27.120"
+        ],
+        "countries": [],
+        "protocols": {
+          "TLS 1.3 /  / AES_128_GCM": 1
+        },
+        "securityState": "secure"
+      }
+    ],
+    "totalLinks": 1,
+    "uniqCountries": 1
+  },
+  "submitter": {
+    "country": "SE"
+  },
+  "task": {
+    "method": "api",
+    "source": "b1c2f290",
+    "tags": [],
+    "time": "2025-12-28T14:50:32.844Z",
+    "url": "https://example.com/",
+    "uuid": "019b6570-74f7-7269-aa5a-48da029c142c",
+    "visibility": "public",
+    "reportURL": "https://urlscan.io/result/019b6570-74f7-7269-aa5a-48da029c142c/",
+    "screenshotURL": "https://urlscan.io/screenshots/019b6570-74f7-7269-aa5a-48da029c142c.png",
+    "domURL": "https://urlscan.io/dom/019b6570-74f7-7269-aa5a-48da029c142c/",
+    "domain": "example.com",
+    "apexDomain": "example.com"
+  },
+  "verdicts": {
+    "overall": {
+      "score": 0,
+      "categories": [],
+      "brands": [],
+      "tags": [],
+      "malicious": false,
+      "hasVerdicts": true
+    },
+    "urlscan": {
+      "score": 0,
+      "categories": [],
+      "brands": [],
+      "tags": [],
+      "malicious": false,
+      "hasVerdicts": false
+    },
+    "engines": {
+      "score": -100,
+      "malicious": false,
+      "categories": [],
+      "enginesTotal": 0,
+      "maliciousTotal": 0,
+      "benignTotal": 0,
+      "maliciousVerdicts": [],
+      "benignVerdicts": [],
+      "hasVerdicts": true,
+      "tags": [
+        "urlscan-ml",
+        "urlscan-ml-0.1.0-1-d9114fd"
+      ]
+    },
+    "community": {
+      "score": 0,
+      "categories": [],
+      "brands": [],
+      "votesTotal": 0,
+      "votesMalicious": 0,
+      "votesBenign": 0,
+      "malicious": false,
+      "hasVerdicts": false
+    }
+  }
+}